Add a 3rd tab to explore human cross annotations.

app.py

@@ -4,6 +4,13 @@ import plotly.express as px
 import plotly.graph_objects as go
 import statsmodels.api as sm
 import random
+import pandas as pd
+import numpy as np
+import plotly.express as px
+import plotly.graph_objects as go
+import matplotlib.pyplot as plt
+import seaborn as sns
+from itertools import combinations
 
 # Set the layout to wide
 st.set_page_config(layout="wide")
@@ -21,6 +28,68 @@ h1, h2, h3, h6{
 st.markdown(center_css, unsafe_allow_html=True)
 
 
+def create_agreement_heatmap(df):
+    # Create a list of unique annotators and sort them by annotator index
+    unique_annotators = sorted(df["annotator_index"].unique())
+
+    # Initialize the agreement matrix and count matrix
+    agreement_matrix = pd.DataFrame(
+        np.nan, index=unique_annotators, columns=unique_annotators
+    )
+    count_matrix = pd.DataFrame(
+        np.zeros((len(unique_annotators), len(unique_annotators))),
+        index=unique_annotators,
+        columns=unique_annotators,
+    )
+
+    # Group by (instruction, output_1, output_2)
+    grouped = df.groupby(["instruction", "output_1", "output_2"])
+
+    for name, group in grouped:
+        # Extract annotators and their preferences
+        annotators = group["annotator_index"].values
+        preferences = group["preference"].values
+
+        # Iterate over all pairs of annotators in the group
+        for (annotator1, pref1), (annotator2, pref2) in combinations(
+            zip(annotators, preferences), 2
+        ):
+            if pref1 == pref2:  # If they agree
+                if pd.isna(agreement_matrix.loc[annotator1, annotator2]):
+                    agreement_matrix.loc[annotator1, annotator2] = 0
+                if pd.isna(agreement_matrix.loc[annotator2, annotator1]):
+                    agreement_matrix.loc[annotator2, annotator1] = 0
+                agreement_matrix.loc[annotator1, annotator2] += 1
+                agreement_matrix.loc[annotator2, annotator1] += 1
+            count_matrix.loc[annotator1, annotator2] += 1
+            count_matrix.loc[annotator2, annotator1] += 1
+
+    # Normalize the agreement matrix by the count matrix
+    for i in unique_annotators:
+        for j in unique_annotators:
+            if count_matrix.loc[i, j] > 0:
+                agreement_matrix.loc[i, j] /= count_matrix.loc[i, j]
+
+    # Plot the heatmap
+    plt.figure(figsize=(10, 10))  # Make the heatmap square
+    sns.heatmap(
+        agreement_matrix,
+        annot=True,
+        fmt=".2f",
+        cmap="PiYG",
+        cbar=True,
+        mask=np.isnan(agreement_matrix),
+        vmin=0.0,
+        vmax=1.0,
+        square=True,
+    )
+    plt.title("Interannotator Agreement Heatmap")
+    plt.xlabel("Annotator")
+    plt.ylabel("Annotator")
+    plt.tight_layout()
+    return agreement_matrix
+
+
 def prep_rankings_table(df, y_column):
     # Create a copy of the dataframe.
     df_copy = df.copy()
@@ -67,6 +136,12 @@ def get_preference_from_rounded_score(score):
     # raise ValueError(f"Invalid score: {score}")
 
 
+def is_unanimous(series):
+    if len(set(series.tolist())) == 1:
+        return True
+    return False
+
+
 def app():
     fixed_model = "gpt4_1106_preview"
 
@@ -77,6 +152,9 @@ def app():
     if "selected_model" not in st.session_state:
         st.session_state.selected_model = "gpt4"
 
+    if "selected_output_human_annotations" not in st.session_state:
+        st.session_state.selected_output_human_annotations = None
+
     if "selected_judge" not in st.session_state:
         st.session_state.selected_judge = None
 
@@ -86,6 +164,12 @@ def app():
     if "instruction_options" not in st.session_state:
         st.session_state.instruction_options = []
 
+    if "instruction_options_human_annotations" not in st.session_state:
+        st.session_state.instruction_options_human_annotations = []
+
+    if "selected_instruction_human_annotations" not in st.session_state:
+        st.session_state.selected_instruction_human_annotations = None
+
     # Function to update the instruction options based on selected dataset
     def update_instruction_options():
         selected_dataset = st.session_state.dataset_selector
@@ -114,6 +198,32 @@ def app():
 
         st.session_state.instruction_options = instruction_options
 
+    def update_instruction_options_human_annotations():
+        selected_dataset = st.session_state.dataset_selector_human_annotations
+        if selected_dataset == "all" or selected_dataset == "NEW":
+            instruction_options = df_human_annotations["instruction"].unique().tolist()
+        elif (
+            selected_dataset == "None"
+            or selected_dataset is None
+            or str(selected_dataset) == ""
+        ):
+            instruction_options = (
+                df_human_annotations[pd.isna(df_human_annotations["dataset"])][
+                    "instruction"
+                ]
+                .unique()
+                .tolist()
+            )
+        else:
+            instruction_options = (
+                df_human_annotations[
+                    df_human_annotations["dataset"] == selected_dataset
+                ]["instruction"]
+                .unique()
+                .tolist()
+            )
+        st.session_state.instruction_options_human_annotations = instruction_options
+
     def update_instruction():
         st.session_state.selected_instruction = st.session_state.instruction_selector
 
@@ -133,10 +243,27 @@ def app():
             st.session_state.instruction_options
         )
 
+    def randomize_selection_human_annotations():
+        st.session_state.dataset_selector_human_annotations = random.choice(
+            ["all"] + df_human_annotations["dataset"].dropna().unique().tolist()
+        )
+        update_instruction_options()
+        st.session_state.selected_instruction_human_annotations = random.choice(
+            st.session_state.instruction_options_human_annotations
+        )
+        st.session_state.selected_output_human_annotations = random.choice(
+            df_human_annotations[
+                df_human_annotations["instruction"]
+                == st.session_state.selected_instruction_human_annotations
+            ]["output_2"]
+            .dropna()
+            .tolist()
+        )
+
     st.title("🦙 AlpacaEval Explorer 🦙")
 
     st.markdown(
-        "### An interactive tool to analyze and explore the data behind the [AlpacaEval Leaderboard](https://tatsu-lab.github.io/alpaca_eval/) in more depth"
+        "###### An interactive tool to analyze and explore the data behind the [AlpacaEval Leaderboard](https://tatsu-lab.github.io/alpaca_eval/) in more depth"
     )
 
     st.markdown(
@@ -149,9 +276,9 @@ def app():
         with st.expander("About AlpacaEval"):
             st.markdown(
                 """- [AlpacaEval](https://github.com/tatsu-lab/alpaca_eval) is an evaluation benchmark to assess the performance of large language models (LLMs).
-- It has high correlation with Chatbot Arena, and is a fast and affordable benchmark for chat LLMs that uses LLMs (specifically GPT-4) to estimate response quality.
+- It has high correlation with [Chatbot Arena](https://chat.lmsys.org/), and is a fast and affordable benchmark for chat LLMs that uses LLMs (specifically GPT-4) to estimate response quality.
 - LLM responses are assessed in a pairwise fashion (arena), where each model's responses are compared to a reference model's responses. 
-- The reference model is GPT-4-1106. The LLM Judge is also GPT-4-1106.
+- All reference responses are generated by GPT-4-1106. The LLM Judge is also GPT-4-1106.
 
 """
             )
@@ -159,9 +286,10 @@ def app():
     with col2:
         with st.expander("About this tool"):
             st.markdown(
-                """- There are 2 main tabs: **Data explorer** and **Length bias explorer**.
-- Use the Data explorer to look at individual pairwise battles between models.
-- Use the Length bias explorer to look at how response lengths affect win rates.
+                """There are 3 main tabs.
+1. Use the **Data explorer** to look at individual pairwise battles between models.
+2. Use the **Length bias explorer** to look at how response lengths affect win rates.
+3. Use the **Human cross annotations** tab to explore the human cross annotations.
 """
             )
 
@@ -169,16 +297,23 @@ def app():
         with st.expander("Motivation"):
             st.markdown(
                 """
-- Several arena-based benchmarks (ours included) have demonstrated that a clear ranking among LLMs can be established, but there is a general dearth of analysis and understanding as to why the rankings are the way they are. For example, it's hard to discern how factors like feel and style
+- Several arena-based benchmarks have demonstrated that a clear ranking among LLMs can be established, but there is a general dearth of analysis and understanding as to why the rankings are the way they are. For example, it's hard to discern how factors like feel and style
 are weighed against correctness.
 - I created this tool to provide a more interactive and intuitive way to explore the data behind the AlpacaEval leaderboard. It allows users to easily compare responses between models, look at individual battles, and analyze how response lengths affect win rates.
-- If you have any feedback on the tool, please reach out on [Twitter](https://twitter.com/justinxzhao)!
+- If you have any feedback on the tool, please reach out.
     """
             )
 
-    outer_tabs = st.tabs(["Data explorer", "Length bias explorer"])
+    outer_tabs = st.tabs(
+        [
+            "Data explorer",
+            "Length bias explorer",
+            "Human cross annotations",
+        ]
+    )
 
     # Load the data
+    df_human_annotations = pd.read_json("data/alpaca_farm_human_crossannotations.json")
     df = pd.read_json("data/model_win_rates.jsonl", lines=True, orient="records")
     # df_responses = pd.read_json("data/df_responses.jsonl", lines=True, orient="records")
     df_response_judging = pd.read_json(
@@ -700,6 +835,167 @@ are weighed against correctness.
                 else:
                     st.error(response_details_dynamic["output_2"])
 
+    with outer_tabs[2]:
+        st.markdown(
+            """The original [AlpacaFarm paper](https://arxiv.org/abs/2305.14387) includes a release of 20K human preferences between a given and reference model on the AlpacaFarm evaluation set. 2.5K of these are cross-annotations (4 humans annotating the same 650 examples). This tab allows you to explore the **human cross-annotations** in more detail."""
+        )
+
+        st.markdown("#### Choose example")
+        st.button(
+            ":game_die: Randomize!",
+            on_click=randomize_selection_human_annotations,
+            type="primary",
+            key="randomize_button_human_annotations",
+        )
+
+        left_col, right_col = st.columns([1, 3])
+
+        st.session_state.selected_dataset_human_annotations = left_col.selectbox(
+            "Select Dataset",
+            ["all"] + df_human_annotations["dataset"].dropna().unique().tolist(),
+            key="dataset_selector_human_annotations",
+            on_change=update_instruction_options_human_annotations,
+        )
+        update_instruction_options_human_annotations()
+        st.session_state.selected_instruction_human_annotations = right_col.selectbox(
+            f"Select Instruction ({len(st.session_state.instruction_options_human_annotations)} unique instructions)",
+            st.session_state.instruction_options_human_annotations,
+            key="instruction_selector_human_annotations",
+            on_change=update_instruction,
+            index=(
+                st.session_state.instruction_options_human_annotations.index(
+                    st.session_state.selected_instruction_human_annotations
+                )
+                if st.session_state.selected_instruction_human_annotations
+                in st.session_state.instruction_options_human_annotations
+                else 0
+            ),
+        )
+
+        st.divider()
+
+        st.markdown(f"#### Selected instruction")
+        st.info(st.session_state.selected_instruction_human_annotations)
+
+        st.divider()
+
+        # Need an output column?
+
+        st.markdown("#### Responses")
+        col1, col2 = st.columns(2)
+
+        with col1:
+            st.selectbox(
+                "Output 1 (reference)",
+                df_human_annotations.loc[
+                    df_human_annotations["instruction"]
+                    == st.session_state.selected_instruction_human_annotations
+                ]["output_1"]
+                .unique()
+                .tolist(),
+                key="output_selector_human_annotations_fuxed",
+                index=0,
+                # label_visibility="collapsed",
+            )
+
+            # Get the response string for the fixed model
+            if st.session_state.selected_instruction_human_annotations:
+                response_details_fixed = df_human_annotations[
+                    (
+                        df_human_annotations["instruction"]
+                        == st.session_state.selected_instruction_human_annotations
+                    )
+                ].iloc[0]
+
+                st.write(
+                    f'Number of words: {len(response_details_fixed["output_1"].split())}'
+                )
+
+                # Display the response string
+                st.info(response_details_fixed["output_1"])
+
+        with col2:
+            st.session_state.selected_output_human_annotations = st.selectbox(
+                "Output 2",
+                df_human_annotations.loc[
+                    df_human_annotations["instruction"]
+                    == st.session_state.selected_instruction_human_annotations
+                ]["output_2"]
+                .dropna()
+                .tolist(),
+                key="output_selector_human_annotations",
+                index=0,
+                # label_visibility="collapsed",
+            )
+
+            # Get the response string for the selected model
+            if (
+                st.session_state.selected_output_human_annotations
+                and st.session_state.selected_instruction_human_annotations
+            ):
+                response_details_dynamic = df_human_annotations[
+                    (
+                        df_human_annotations["instruction"]
+                        == st.session_state.selected_instruction_human_annotations
+                    )
+                    & (
+                        df_human_annotations["output_2"]
+                        == st.session_state.selected_output_human_annotations
+                    )
+                ].iloc[0]
+
+                st.write(
+                    f'Number of words: {len(response_details_dynamic["output_2"].split())}'
+                )
+                st.info(response_details_dynamic["output_2"])
+
+        st.divider()
+
+        # Judging details.
+        st.markdown(f"#### Human Judging")
+
+        col1, col2 = st.columns(2)
+
+        with col1:
+
+            judging_details = df_human_annotations[
+                (df_human_annotations["output_1"] == response_details_fixed["output_1"])
+                & (
+                    df_human_annotations["output_2"]
+                    == response_details_dynamic["output_2"]
+                )
+            ]
+            judging_details["assigned_preference"] = judging_details[
+                "preference"
+            ].apply(get_preference_from_rounded_score)
+            is_unanimous_value = is_unanimous(judging_details["preference"])
+            st.write("**Unanimous?** ", is_unanimous_value)
+
+            # Draw a histogram of preference.
+            fig = px.histogram(
+                judging_details,
+                x="assigned_preference",
+            )
+            fig.update_layout(xaxis_title="Preference")
+            st.plotly_chart(fig)
+
+            with st.expander("Data details"):
+                st.dataframe(
+                    judging_details[["annotator_index", "assigned_preference"]],
+                    hide_index=True,
+                )
+
+        # Generate the heatmap figure
+        with col2:
+            agreement_matrix = create_agreement_heatmap(df_human_annotations)
+            # st.write(
+            #     f"**Overall interannotator agreement:** {agreement_matrix.mean().mean():.3f}"
+            # )
+            with st.expander(
+                f"**Overall interannotator agreement:** {agreement_matrix.mean().mean():.3f}"
+            ):
+                st.pyplot(plt)
+
 
 if __name__ == "__main__":
     app()