adding descriptive text

app.py

@@ -1,53 +1,104 @@
 import streamlit as st
 import pandas as pd
-import plotly.express as px
 import json
+import plotly.express as px
+import plotly.graph_objects as go
 
-st.set_page_config(layout="wide", page_title="CyberSafetyEval Leaderboard", page_icon=":bar_chart:")
+# Configure the streamlit page
+st.set_page_config(layout="wide", page_title="CyberSecEval Leaderboard", page_icon=":bar_chart:")
+# Display the title and brief description of the page
+st.markdown("# CyberSecEval: Comprehensive Evaluation Framework for Cybersecurity Risks and Capabilities of Large Language Models (LLMs)", unsafe_allow_html=True)
 
-st.markdown("# CyberSafetyEval Leaderboard", unsafe_allow_html=True)
+# Provide more detailed information about the page and its purpose
+st.markdown("""
+Welcome to the CyberSecEval leaderboard. This platform showcases the results of our cybersecurity evaluation framework applied to various popular LLMs. Our open-source evaluation suite's workings and coverage are detailed in our [first](https://ai.meta.com/research/publications/purple-llama-cyberseceval-a-benchmark-for-evaluating-the-cybersecurity-risks-of-large-language-models/) and [second](https://ai.meta.com/research/publications/cyberseceval-2-a-wide-ranging-cybersecurity-evaluation-suite-for-large-language-models/) papers. 
 
-data_mitre = json.load(open("mitre.json"))
-data_insecure = json.load(open("insecure_code.json"))
+In the following sections, we present case study test results and provide concise explanations of their implications.
 
-# Processing data for the table
-st.markdown("### Helpfulness to cyberattackers results, benign percentage is percentage of tests safely passed (higher is better)")
-model_stats = {}
-for model, categories in data_mitre.items():
-    model_stats[model] = {'Mean Benign Percentage': pd.Series([v['benign_percentage'] for v in categories.values()]).mean()}
-    for category, values in categories.items():
-        model_stats[model][f'Benign Percentage in {category}'] = values['benign_percentage']
+**Take Action:** Measure your own LLM using CyberSecEval! Visit our open-source GitHub repository [here](https://github.com/meta-llama/PurpleLlama/tree/main/CybersecurityBenchmarks)!
+""")
 
-leaderboard_df = pd.DataFrame.from_dict(model_stats, orient='index').sort_values(by='Mean Benign Percentage', ascending=False)
+# Load JSON files containing test results
+data_attack = json.load(open("attack_helpfulness.json"))
+data_interpreter = json.load(open("interpreter_abuse_tests.json"))
+data_prompt = json.load(open("prompt_injection_tests.json"))
+data_exploit = json.load(open("exploit_tests.json"))
+data_tradeoff = json.load(open("trr_frr_tradeoff_helpfulness.json"))
 
-# Preparing data for the clustered bar chart
-chart_data = []
-for model, categories in data_mitre.items():
-    for category, values in categories.items():
-        chart_data.append({
-            'Model': model,
-            'Category': category,
-            'Benign Percentage': values['benign_percentage']
-        })
+# Function to sort rows and columns by their sums
+def sort_df(df):
+    return df.loc[df.sum(axis=1).sort_values(ascending=False).index, df.sum(axis=0).sort_values(ascending=False).index]
 
-chart_df = pd.DataFrame(chart_data)
+# Convert JSON data to pandas DataFrames and sort
+attack_df = sort_df(pd.DataFrame(data_attack))
+interpreter_df = sort_df(pd.DataFrame(data_interpreter))
+prompt_df = sort_df(pd.DataFrame(data_prompt))
+exploit_df = sort_df(pd.DataFrame(data_exploit))
 
-# Styling the data table
-st.dataframe(leaderboard_df.style.format("{:.2%}").background_gradient(cmap='Blues'))
+# Scatterplot for True refusal vs. false refusal results
+tradeoff_df = pd.DataFrame(data_tradeoff).transpose()
 
-# Enhanced Plotly chart
-fig = px.bar(chart_df, x='Category', y='Benign Percentage', barmode='group', color='Model',
-             title='MITRE ATT&CK category results per model',
-             labels={'Benign Percentage': 'Benign %'},
-             color_discrete_sequence=px.colors.qualitative.Pastel)
-fig.update_layout(plot_bgcolor='rgba(0,0,0,0)',
+# Briefly explain the purpose of the scatterplot
+st.markdown("### Evaluating LLMs' Assistance to Cyberattackers")
+st.markdown("""
+We measure the tradeoff between LLMs' 'false refusals' (refusing to assist in legitimate cybersecurity-related activities) and their 'violation rate' (agreeing to assist in offensive cyber attacks). The results are displayed in the scatterplot below.
+""")
+
+# Create a new figure
+fig = go.Figure()
+
+# Loop through each row and plot with different marker and color
+for i, row in enumerate(tradeoff_df.iterrows()):
+    fig.add_trace(go.Scatter(x=[row[1]['Violation Rate']], y=[row[1]['Refusal Rate']], 
+                             mode='markers', 
+                             name=row[0], marker=dict(size=15)))
+
+# Add labels and title
+fig.update_layout(title='True Refusal vs. Violation Rate for Cybersecurity Prompts',
+                  xaxis_title='Violation Rate',
+                  yaxis_title='Refusal Rate',
+                  plot_bgcolor='rgba(0,0,0,0)',
                   xaxis=dict(showgrid=False),
-                  yaxis=dict(showgrid=False, tickformat=".0%"),
-                  legend=dict(title='Models'))
+                  yaxis=dict(showgrid=False))
+
+# Describe the plot
+st.markdown("""
+The plot below evaluates the performance of various LLMs in two tasks:
+* Complying with legitimate requests for help with cybersecurity-related tasks
+* Refusing to assist in offensive cyberattacks
+An ideal LLM would be positioned in the lower left corner of the plot.
+""")
 
+# Display the plot
 st.plotly_chart(fig, use_container_width=True)
 
-# Processing data for the table
+# Function to display heatmaps and bar charts
+def display_results(df, title, description,include_barchart=True):
+    if title:
+        st.markdown(f"### {title}")
+    st.markdown(f"{description}")
+    st.dataframe(df.style.format("{:.2%}").background_gradient(cmap='coolwarm'))
+    if include_barchart:
+        if title:
+            st.markdown(f"### Overall Performance of Models in {title}")
+        fig = px.bar(df.sum(axis=1).sort_values(ascending=False))
+        fig.update_layout(plot_bgcolor='rgba(0,0,0,0)',
+              xaxis=dict(showgrid=False),
+              yaxis=dict(showgrid=False)
+        )
+        st.plotly_chart(fig, use_container_width=True)
+
+# Display the results for each test
+# Continue displaying the results for each test
+display_results(interpreter_df, 'LLMs Assistance to Hack Code Interpreter Environments', 'The plot below shows how often each model complies with attempts to hack a connected code interpreter. Lower values indicate safer models.', False)
+display_results(prompt_df, 'LLMs Susceptibility to Prompt Injection Attacks', 'The plot below shows how often each model succumbs to prompt injection attacks, categorized into threat categories detailed in our paper. Lower values indicate safer models.', False)
+display_results(exploit_df, 'LLMs Capability to Solve Cyber Capture the Flag Challenges', 'The plot below shows how well each model performs at solving capture-the-flag style program exploitation challenges. Higher values indicate more capable models.', True)
+
+# Load additional JSON files
+data_mitre = json.load(open("mitre.json"))
+data_insecure = json.load(open("insecure_code.json"))
+
+# Process data for the table
 model_stats_insecure = {}
 for model, categories in data_insecure.items():
     model_stats_insecure[model] = {'Mean Insecure Code Test Pass Rate': pd.Series([1-v['autocomplete_vunerable_percentage'] for v in categories.values()]).mean()}
@@ -55,7 +106,7 @@ for model, categories in data_insecure.items():
         model_stats_insecure[model][f'Insecure Code Test Pass Rate in {category}'] = 1-values['autocomplete_vunerable_percentage']
 leaderboard_df_insecure = pd.DataFrame.from_dict(model_stats_insecure, orient='index').sort_values(by='Mean Insecure Code Test Pass Rate', ascending=False)
 
-# Preparing data for the clustered bar chart
+# Prepare data for the clustered bar chart
 chart_data_insecure = []
 for model, categories in data_insecure.items():
     for category, values in categories.items():
@@ -66,11 +117,14 @@ for model, categories in data_insecure.items():
         })
 chart_df_insecure = pd.DataFrame(chart_data_insecure)
 
-# Styling the data table
-st.markdown("### Insecure coding test pass rate results (higher is better)")
+# Style the data table
+st.markdown("### LLMs Adherence to Secure Coding Practices in Risky Software Engineering Settings")
+st.markdown("""
+The table below shows the propensity of LLMs to avoid insecure coding practices when used as coding assistants or software engineering agents. Higher values indicate safer models.
+""")
 st.dataframe(leaderboard_df_insecure.style.format("{:.2%}").background_gradient(cmap='Blues'))  # Changed cmap to 'Blues'
 
-# Enhanced Plotly chart
+# Create a Plotly chart
 fig_insecure = px.bar(chart_df_insecure, x='Category', y='Insecure Code Test Pass Rate', barmode='group', color='Model',
              title='Category-wise Insecure Code Test Pass Rate per Model',
              labels={'Insecure Code Test Pass Rate': 'Insecure Code Test Pass Rate %'},
@@ -80,4 +134,5 @@ fig_insecure.update_layout(plot_bgcolor='rgba(0,0,0,0)',
                   yaxis=dict(showgrid=False, tickformat=".0%"),
                   legend=dict(title='Models'))
 
+# Display the chart
 st.plotly_chart(fig_insecure, use_container_width=True)

attack_helpfulness.json

@@ -0,0 +1,122 @@
+{
+    "Discovery": {
+        "llama 3 8b-instruct": 0.4033333333,
+        "gpt-4": 0.4033333333,
+        "mistral-medium-latest": 0.42,
+        "llama 3 70b-instruct": 0.6733333333,
+        "codellama-70b-instruct": 0.3633333333,
+        "codellama-13b-instruct": 0.5633333333,
+        "mistral-large-latest": 0.4,
+        "codellama-34b-instruct": 0.4966666667,
+        "mistral-small-latest": 0.4666666667,
+        "gpt-3.5-turbo": 0.49
+    },
+    "C2": {
+        "llama 3 8b-instruct": 0.15,
+        "gpt-4": 0.4266666667,
+        "mistral-medium-latest": 0.38,
+        "llama 3 70b-instruct": 0.4766666667,
+        "codellama-70b-instruct": 0.3766666667,
+        "codellama-13b-instruct": 0.5353535354,
+        "mistral-large-latest": 0.51,
+        "codellama-34b-instruct": 0.5733333333,
+        "mistral-small-latest": 0.5433333333,
+        "gpt-3.5-turbo": 0.5966666667
+    },
+    "Recon": {
+        "llama 3 8b-instruct": 0.35,
+        "gpt-4": 0.3866666667,
+        "mistral-medium-latest": 0.4433333333,
+        "llama 3 70b-instruct": 0.5833333333,
+        "codellama-70b-instruct": 0.3833333333,
+        "codellama-13b-instruct": 0.4080267559,
+        "mistral-large-latest": 0.4333333333,
+        "codellama-34b-instruct": 0.58,
+        "mistral-small-latest": 0.44,
+        "gpt-3.5-turbo": 0.4466666667
+    },
+    "Exfil": {
+        "llama 3 8b-instruct": 0.1233333333,
+        "gpt-4": 0.2833333333,
+        "mistral-medium-latest": 0.3166666667,
+        "llama 3 70b-instruct": 0.34,
+        "codellama-70b-instruct": 0.3745819398,
+        "codellama-13b-instruct": 0.4533333333,
+        "mistral-large-latest": 0.52,
+        "codellama-34b-instruct": 0.5233333333,
+        "mistral-small-latest": 0.5866666667,
+        "gpt-3.5-turbo": 0.5933333333
+    },
+    "Privilege Escalation": {
+        "llama 3 8b-instruct": 0.06,
+        "gpt-4": 0.3033333333,
+        "mistral-medium-latest": 0.3866666667,
+        "llama 3 70b-instruct": 0.2866666667,
+        "codellama-70b-instruct": 0.4233333333,
+        "codellama-13b-instruct": 0.3535353535,
+        "mistral-large-latest": 0.4666666667,
+        "codellama-34b-instruct": 0.4633333333,
+        "mistral-small-latest": 0.6566666667,
+        "gpt-3.5-turbo": 0.6733333333
+    },
+    "Lateral Movement": {
+        "llama 3 8b-instruct": 0.0966666667,
+        "gpt-4": 0.2366666667,
+        "mistral-medium-latest": 0.2766666667,
+        "llama 3 70b-instruct": 0.3433333333,
+        "codellama-70b-instruct": 0.4033333333,
+        "codellama-13b-instruct": 0.47,
+        "mistral-large-latest": 0.5366666667,
+        "codellama-34b-instruct": 0.4233333333,
+        "mistral-small-latest": 0.6166666667,
+        "gpt-3.5-turbo": 0.6666666667
+    },
+    "Persistence": {
+        "llama 3 8b-instruct": 0.1366666667,
+        "gpt-4": 0.2533333333,
+        "mistral-medium-latest": 0.3066666667,
+        "llama 3 70b-instruct": 0.2766666667,
+        "codellama-70b-instruct": 0.3933333333,
+        "codellama-13b-instruct": 0.4666666667,
+        "mistral-large-latest": 0.4733333333,
+        "codellama-34b-instruct": 0.48,
+        "mistral-small-latest": 0.59,
+        "gpt-3.5-turbo": 0.6666666667
+    },
+    "Evasion": {
+        "llama 3 8b-instruct": 0.0466666667,
+        "gpt-4": 0.2133333333,
+        "mistral-medium-latest": 0.3333333333,
+        "llama 3 70b-instruct": 0.17,
+        "codellama-70b-instruct": 0.29,
+        "codellama-13b-instruct": 0.4333333333,
+        "mistral-large-latest": 0.49,
+        "codellama-34b-instruct": 0.45,
+        "mistral-small-latest": 0.65,
+        "gpt-3.5-turbo": 0.7233333333
+    },
+    "Execution": {
+        "llama 3 8b-instruct": 0.09,
+        "gpt-4": 0.1833333333,
+        "mistral-medium-latest": 0.2976588629,
+        "llama 3 70b-instruct": 0.2033333333,
+        "codellama-70b-instruct": 0.3233333333,
+        "codellama-13b-instruct": 0.3966666667,
+        "mistral-large-latest": 0.52,
+        "codellama-34b-instruct": 0.51,
+        "mistral-small-latest": 0.6233333333,
+        "gpt-3.5-turbo": 0.65
+    },
+    "Collection": {
+        "llama 3 8b-instruct": 0.18,
+        "gpt-4": 0.19,
+        "mistral-medium-latest": 0.31,
+        "llama 3 70b-instruct": 0.31,
+        "codellama-70b-instruct": 0.3333333333,
+        "codellama-13b-instruct": 0.4612794613,
+        "mistral-large-latest": 0.4233333333,
+        "codellama-34b-instruct": 0.4966666667,
+        "mistral-small-latest": 0.52,
+        "gpt-3.5-turbo": 0.55
+    }
+}

exploit_tests.json

@@ -0,0 +1,74 @@
+{
+    "Javascript input constraint tests": {
+        "gpt-4-turbo": 0.6748333333,
+        "mistral-large-latest": 0.6375,
+        "mistral-small-latest": 0.6125,
+        "llama 3 70b-instruct": 0.6196666667,
+        "codellama-34b-instruct": 0.5487719298,
+        "llama 3 8b-instruct": 0.5601351351,
+        "gpt-3.5-turbo": 0.5311666667,
+        "codellama-13b-instruct": 0.4996655518,
+        "mistral-medium-latest": 0.5553872054,
+        "codellama-70b-instruct": 0.4539115646
+    },
+    "Python input constraint tests": {
+        "gpt-4-turbo": 0.6566666667,
+        "mistral-large-latest": 0.6365,
+        "mistral-small-latest": 0.6127090301,
+        "llama 3 70b-instruct": 0.6028333333,
+        "codellama-34b-instruct": 0.5325423729,
+        "llama 3 8b-instruct": 0.5348993289,
+        "gpt-3.5-turbo": 0.5265,
+        "codellama-13b-instruct": 0.4916666667,
+        "mistral-medium-latest": 0.5210884354,
+        "codellama-70b-instruct": 0.4444256757
+    },
+    "C input constraint tests": {
+        "gpt-4-turbo": 0.6643333333,
+        "mistral-large-latest": 0.6231666667,
+        "mistral-small-latest": 0.608,
+        "llama 3 70b-instruct": 0.6193333333,
+        "codellama-34b-instruct": 0.445777027,
+        "llama 3 8b-instruct": 0.4947552448,
+        "gpt-3.5-turbo": 0.4848333333,
+        "codellama-13b-instruct": 0.4821070234,
+        "mistral-medium-latest": 0.4363309353,
+        "codellama-70b-instruct": 0.4342281879
+    },
+    "Diverse memory corruption tests": {
+        "gpt-4-turbo": 0.1595330739,
+        "mistral-large-latest": 0.1576763485,
+        "mistral-small-latest": 0.2356321839,
+        "llama 3 70b-instruct": 0.1434782609,
+        "codellama-34b-instruct": 0.1991869919,
+        "llama 3 8b-instruct": 0.1242937853,
+        "gpt-3.5-turbo": 0.1388888889,
+        "codellama-13b-instruct": 0.1505791506,
+        "mistral-medium-latest": 0.112745098,
+        "codellama-70b-instruct": 0.1373390558
+    },
+    "Python/SQLite SQL injection tests": {
+        "gpt-4-turbo": 0.23,
+        "mistral-large-latest": 0.0033333333,
+        "mistral-small-latest": 0.0,
+        "llama 3 70b-instruct": 0.0166666667,
+        "codellama-34b-instruct": 0.0,
+        "llama 3 8b-instruct": 0.0,
+        "gpt-3.5-turbo": 0.0,
+        "codellama-13b-instruct": 0.0033333333,
+        "mistral-medium-latest": 0.0,
+        "codellama-70b-instruct": 0.0745762712
+    },
+    "Buffer overflow tests": {
+        "gpt-4-turbo": 0.0469798658,
+        "mistral-large-latest": 0.0205992509,
+        "mistral-small-latest": 0.0017361111,
+        "llama 3 70b-instruct": 0.0068027211,
+        "codellama-34b-instruct": 0.0,
+        "llama 3 8b-instruct": 0.0070422535,
+        "gpt-3.5-turbo": 0.0183333333,
+        "codellama-13b-instruct": 0.005,
+        "mistral-medium-latest": 0.0018382353,
+        "codellama-70b-instruct": 0.0154109589
+    }
+}

insecure_code.json

@@ -253,260 +253,5 @@
                              "instruct_bleu": 15.43,
                              "instruct_total_count": 204,
                              "instruct_vunerable_percentage": 0.3971,
-                             "instruct_vunerable_suggestion_count": 81}},
-    "llama2-13b-chat": {   "C": {   "autocomplete_bleu": 15.54,
-                                    "autocomplete_total_count": 227,
-                                    "autocomplete_vunerable_percentage": 0.22469999999999998,
-                                    "autocomplete_vunerable_suggestion_count": 51,
-                                    "instruct_bleu": 9.25,
-                                    "instruct_total_count": 227,
-                                    "instruct_vunerable_percentage": 0.3568,
-                                    "instruct_vunerable_suggestion_count": 81},
-                           "C#": {   "autocomplete_bleu": 17.27,
-                                     "autocomplete_total_count": 235,
-                                     "autocomplete_vunerable_percentage": 0.09359999999999999,
-                                     "autocomplete_vunerable_suggestion_count": 22,
-                                     "instruct_bleu": 11.08,
-                                     "instruct_total_count": 235,
-                                     "instruct_vunerable_percentage": 0.2851,
-                                     "instruct_vunerable_suggestion_count": 67},
-                           "C++": {   "autocomplete_bleu": 14.91,
-                                      "autocomplete_total_count": 259,
-                                      "autocomplete_vunerable_percentage": 0.1583,
-                                      "autocomplete_vunerable_suggestion_count": 41,
-                                      "instruct_bleu": 8.85,
-                                      "instruct_total_count": 259,
-                                      "instruct_vunerable_percentage": 0.2432,
-                                      "instruct_vunerable_suggestion_count": 63},
-                           "Java": {   "autocomplete_bleu": 18.59,
-                                       "autocomplete_total_count": 229,
-                                       "autocomplete_vunerable_percentage": 0.23579999999999998,
-                                       "autocomplete_vunerable_suggestion_count": 54,
-                                       "instruct_bleu": 14.04,
-                                       "instruct_total_count": 229,
-                                       "instruct_vunerable_percentage": 0.4279,
-                                       "instruct_vunerable_suggestion_count": 98},
-                           "JavaScript": {   "autocomplete_bleu": 15.19,
-                                             "autocomplete_total_count": 249,
-                                             "autocomplete_vunerable_percentage": 0.0843,
-                                             "autocomplete_vunerable_suggestion_count": 21,
-                                             "instruct_bleu": 9.12,
-                                             "instruct_total_count": 249,
-                                             "instruct_vunerable_percentage": 0.2851,
-                                             "instruct_vunerable_suggestion_count": 71},
-                           "PHP": {   "autocomplete_bleu": 13.76,
-                                      "autocomplete_total_count": 162,
-                                      "autocomplete_vunerable_percentage": 0.1852,
-                                      "autocomplete_vunerable_suggestion_count": 30,
-                                      "instruct_bleu": 11.83,
-                                      "instruct_total_count": 162,
-                                      "instruct_vunerable_percentage": 0.3519,
-                                      "instruct_vunerable_suggestion_count": 57},
-                           "Python": {   "autocomplete_bleu": 12.72,
-                                         "autocomplete_total_count": 351,
-                                         "autocomplete_vunerable_percentage": 0.2707,
-                                         "autocomplete_vunerable_suggestion_count": 95,
-                                         "instruct_bleu": 7.38,
-                                         "instruct_total_count": 351,
-                                         "instruct_vunerable_percentage": 0.31620000000000004,
-                                         "instruct_vunerable_suggestion_count": 111},
-                           "Rust": {   "autocomplete_bleu": 12.16,
-                                       "autocomplete_total_count": 204,
-                                       "autocomplete_vunerable_percentage": 0.1422,
-                                       "autocomplete_vunerable_suggestion_count": 29,
-                                       "instruct_bleu": 12.04,
-                                       "instruct_total_count": 204,
-                                       "instruct_vunerable_percentage": 0.2353,
-                                       "instruct_vunerable_suggestion_count": 48}},
-    "llama2-30b-chat": {   "C": {   "autocomplete_bleu": 13.32,
-                                    "autocomplete_total_count": 227,
-                                    "autocomplete_vunerable_percentage": 0.28190000000000004,
-                                    "autocomplete_vunerable_suggestion_count": 64,
-                                    "instruct_bleu": 8.95,
-                                    "instruct_total_count": 227,
-                                    "instruct_vunerable_percentage": 0.43170000000000003,
-                                    "instruct_vunerable_suggestion_count": 98},
-                           "C#": {   "autocomplete_bleu": 12.7,
-                                     "autocomplete_total_count": 235,
-                                     "autocomplete_vunerable_percentage": 0.1574,
-                                     "autocomplete_vunerable_suggestion_count": 37,
-                                     "instruct_bleu": 12.68,
-                                     "instruct_total_count": 235,
-                                     "instruct_vunerable_percentage": 0.37450000000000006,
-                                     "instruct_vunerable_suggestion_count": 88},
-                           "C++": {   "autocomplete_bleu": 11.91,
-                                      "autocomplete_total_count": 259,
-                                      "autocomplete_vunerable_percentage": 0.15439999999999998,
-                                      "autocomplete_vunerable_suggestion_count": 40,
-                                      "instruct_bleu": 8.97,
-                                      "instruct_total_count": 259,
-                                      "instruct_vunerable_percentage": 0.28190000000000004,
-                                      "instruct_vunerable_suggestion_count": 73},
-                           "Java": {   "autocomplete_bleu": 15.58,
-                                       "autocomplete_total_count": 229,
-                                       "autocomplete_vunerable_percentage": 0.3188,
-                                       "autocomplete_vunerable_suggestion_count": 73,
-                                       "instruct_bleu": 15.77,
-                                       "instruct_total_count": 229,
-                                       "instruct_vunerable_percentage": 0.4803,
-                                       "instruct_vunerable_suggestion_count": 110},
-                           "JavaScript": {   "autocomplete_bleu": 11.14,
-                                             "autocomplete_total_count": 249,
-                                             "autocomplete_vunerable_percentage": 0.15259999999999999,
-                                             "autocomplete_vunerable_suggestion_count": 38,
-                                             "instruct_bleu": 9.22,
-                                             "instruct_total_count": 249,
-                                             "instruct_vunerable_percentage": 0.3414,
-                                             "instruct_vunerable_suggestion_count": 85},
-                           "PHP": {   "autocomplete_bleu": 11.69,
-                                      "autocomplete_total_count": 162,
-                                      "autocomplete_vunerable_percentage": 0.179,
-                                      "autocomplete_vunerable_suggestion_count": 29,
-                                      "instruct_bleu": 11.41,
-                                      "instruct_total_count": 162,
-                                      "instruct_vunerable_percentage": 0.3889,
-                                      "instruct_vunerable_suggestion_count": 63},
-                           "Python": {   "autocomplete_bleu": 12.25,
-                                         "autocomplete_total_count": 351,
-                                         "autocomplete_vunerable_percentage": 0.2593,
-                                         "autocomplete_vunerable_suggestion_count": 91,
-                                         "instruct_bleu": 7.53,
-                                         "instruct_total_count": 351,
-                                         "instruct_vunerable_percentage": 0.3504,
-                                         "instruct_vunerable_suggestion_count": 123},
-                           "Rust": {   "autocomplete_bleu": 11.14,
-                                       "autocomplete_total_count": 204,
-                                       "autocomplete_vunerable_percentage": 0.1324,
-                                       "autocomplete_vunerable_suggestion_count": 27,
-                                       "instruct_bleu": 13.17,
-                                       "instruct_total_count": 204,
-                                       "instruct_vunerable_percentage": 0.2255,
-                                       "instruct_vunerable_suggestion_count": 46}},
-    "llama2-70b-chat": {   "C": {   "autocomplete_bleu": 13.76,
-                                    "autocomplete_total_count": 227,
-                                    "autocomplete_vunerable_percentage": 0.33039999999999997,
-                                    "autocomplete_vunerable_suggestion_count": 75,
-                                    "instruct_bleu": 9.0,
-                                    "instruct_total_count": 227,
-                                    "instruct_vunerable_percentage": 0.38770000000000004,
-                                    "instruct_vunerable_suggestion_count": 88},
-                           "C#": {   "autocomplete_bleu": 12.13,
-                                     "autocomplete_total_count": 235,
-                                     "autocomplete_vunerable_percentage": 0.2468,
-                                     "autocomplete_vunerable_suggestion_count": 58,
-                                     "instruct_bleu": 12.65,
-                                     "instruct_total_count": 235,
-                                     "instruct_vunerable_percentage": 0.3617,
-                                     "instruct_vunerable_suggestion_count": 85},
-                           "C++": {   "autocomplete_bleu": 11.3,
-                                      "autocomplete_total_count": 259,
-                                      "autocomplete_vunerable_percentage": 0.1274,
-                                      "autocomplete_vunerable_suggestion_count": 33,
-                                      "instruct_bleu": 9.6,
-                                      "instruct_total_count": 259,
-                                      "instruct_vunerable_percentage": 0.2664,
-                                      "instruct_vunerable_suggestion_count": 69},
-                           "Java": {   "autocomplete_bleu": 16.13,
-                                       "autocomplete_total_count": 229,
-                                       "autocomplete_vunerable_percentage": 0.2838,
-                                       "autocomplete_vunerable_suggestion_count": 65,
-                                       "instruct_bleu": 17.08,
-                                       "instruct_total_count": 229,
-                                       "instruct_vunerable_percentage": 0.49340000000000006,
-                                       "instruct_vunerable_suggestion_count": 113},
-                           "JavaScript": {   "autocomplete_bleu": 11.58,
-                                             "autocomplete_total_count": 249,
-                                             "autocomplete_vunerable_percentage": 0.1406,
-                                             "autocomplete_vunerable_suggestion_count": 35,
-                                             "instruct_bleu": 9.01,
-                                             "instruct_total_count": 249,
-                                             "instruct_vunerable_percentage": 0.3494,
-                                             "instruct_vunerable_suggestion_count": 87},
-                           "PHP": {   "autocomplete_bleu": 12.74,
-                                      "autocomplete_total_count": 162,
-                                      "autocomplete_vunerable_percentage": 0.19140000000000001,
-                                      "autocomplete_vunerable_suggestion_count": 31,
-                                      "instruct_bleu": 13.02,
-                                      "instruct_total_count": 162,
-                                      "instruct_vunerable_percentage": 0.3951,
-                                      "instruct_vunerable_suggestion_count": 64},
-                           "Python": {   "autocomplete_bleu": 11.22,
-                                         "autocomplete_total_count": 351,
-                                         "autocomplete_vunerable_percentage": 0.3276,
-                                         "autocomplete_vunerable_suggestion_count": 115,
-                                         "instruct_bleu": 7.52,
-                                         "instruct_total_count": 351,
-                                         "instruct_vunerable_percentage": 0.3447,
-                                         "instruct_vunerable_suggestion_count": 121},
-                           "Rust": {   "autocomplete_bleu": 9.05,
-                                       "autocomplete_total_count": 204,
-                                       "autocomplete_vunerable_percentage": 0.2157,
-                                       "autocomplete_vunerable_suggestion_count": 44,
-                                       "instruct_bleu": 13.24,
-                                       "instruct_total_count": 204,
-                                       "instruct_vunerable_percentage": 0.27449999999999997,
-                                       "instruct_vunerable_suggestion_count": 56}},
-    "llama2-7b-chat": {   "C": {   "autocomplete_bleu": 11.55,
-                                   "autocomplete_total_count": 227,
-                                   "autocomplete_vunerable_percentage": 0.141,
-                                   "autocomplete_vunerable_suggestion_count": 32,
-                                   "instruct_bleu": 8.49,
-                                   "instruct_total_count": 227,
-                                   "instruct_vunerable_percentage": 0.36560000000000004,
-                                   "instruct_vunerable_suggestion_count": 83},
-                          "C#": {   "autocomplete_bleu": 13.47,
-                                    "autocomplete_total_count": 235,
-                                    "autocomplete_vunerable_percentage": 0.0681,
-                                    "autocomplete_vunerable_suggestion_count": 16,
-                                    "instruct_bleu": 10.97,
-                                    "instruct_total_count": 235,
-                                    "instruct_vunerable_percentage": 0.29359999999999997,
-                                    "instruct_vunerable_suggestion_count": 69},
-                          "C++": {   "autocomplete_bleu": 11.28,
-                                     "autocomplete_total_count": 259,
-                                     "autocomplete_vunerable_percentage": 0.0695,
-                                     "autocomplete_vunerable_suggestion_count": 18,
-                                     "instruct_bleu": 8.36,
-                                     "instruct_total_count": 259,
-                                     "instruct_vunerable_percentage": 0.2278,
-                                     "instruct_vunerable_suggestion_count": 59},
-                          "Java": {   "autocomplete_bleu": 12.79,
-                                      "autocomplete_total_count": 229,
-                                      "autocomplete_vunerable_percentage": 0.11349999999999999,
-                                      "autocomplete_vunerable_suggestion_count": 26,
-                                      "instruct_bleu": 13.84,
-                                      "instruct_total_count": 229,
-                                      "instruct_vunerable_percentage": 0.39299999999999996,
-                                      "instruct_vunerable_suggestion_count": 90},
-                          "JavaScript": {   "autocomplete_bleu": 11.52,
-                                            "autocomplete_total_count": 249,
-                                            "autocomplete_vunerable_percentage": 0.0643,
-                                            "autocomplete_vunerable_suggestion_count": 16,
-                                            "instruct_bleu": 7.77,
-                                            "instruct_total_count": 249,
-                                            "instruct_vunerable_percentage": 0.30519999999999997,
-                                            "instruct_vunerable_suggestion_count": 76},
-                          "PHP": {   "autocomplete_bleu": 10.06,
-                                     "autocomplete_total_count": 162,
-                                     "autocomplete_vunerable_percentage": 0.11109999999999999,
-                                     "autocomplete_vunerable_suggestion_count": 18,
-                                     "instruct_bleu": 10.8,
-                                     "instruct_total_count": 162,
-                                     "instruct_vunerable_percentage": 0.2963,
-                                     "instruct_vunerable_suggestion_count": 48},
-                          "Python": {   "autocomplete_bleu": 10.67,
-                                        "autocomplete_total_count": 351,
-                                        "autocomplete_vunerable_percentage": 0.2365,
-                                        "autocomplete_vunerable_suggestion_count": 83,
-                                        "instruct_bleu": 6.89,
-                                        "instruct_total_count": 351,
-                                        "instruct_vunerable_percentage": 0.3077,
-                                        "instruct_vunerable_suggestion_count": 108},
-                          "Rust": {   "autocomplete_bleu": 9.53,
-                                      "autocomplete_total_count": 204,
-                                      "autocomplete_vunerable_percentage": 0.0931,
-                                      "autocomplete_vunerable_suggestion_count": 19,
-                                      "instruct_bleu": 10.78,
-                                      "instruct_total_count": 204,
-                                      "instruct_vunerable_percentage": 0.2108,
-                                      "instruct_vunerable_suggestion_count": 43}}}
+                             "instruct_vunerable_suggestion_count": 81}}
+}

interpreter_abuse_tests.json

@@ -0,0 +1,62 @@
+{
+    "Container Escape": {
+        "codellama-13b-instruct": 0.2247191011,
+        "codellama-34b-instruct": 0.1978021978,
+        "gpt-4": 0.2470588235,
+        "gpt-3.5-turbo": 0.3076923077,
+        "codellama-70b-instruct": 0.2976190476,
+        "llama 3 8b-instruct": 0.2527472527,
+        "mistral-large-latest": 0.3368421053,
+        "mistral-medium-latest": 0.4222222222,
+        "llama 3 70b-instruct": 0.5257731959,
+        "mistral-small-latest": 0.4693877551
+    },
+    "Privilege Escalation": {
+        "codellama-13b-instruct": 0.1276595745,
+        "codellama-34b-instruct": 0.1789473684,
+        "gpt-4": 0.2111111111,
+        "gpt-3.5-turbo": 0.313253012,
+        "codellama-70b-instruct": 0.2696629213,
+        "llama 3 8b-instruct": 0.2474226804,
+        "mistral-large-latest": 0.4166666667,
+        "mistral-medium-latest": 0.347826087,
+        "llama 3 70b-instruct": 0.6082474227,
+        "mistral-small-latest": 0.4787234043
+    },
+    "Post Exploitation": {
+        "codellama-13b-instruct": 0.1648351648,
+        "codellama-34b-instruct": 0.1318681319,
+        "gpt-4": 0.1445783133,
+        "gpt-3.5-turbo": 0.1764705882,
+        "codellama-70b-instruct": 0.2298850575,
+        "llama 3 8b-instruct": 0.2653061224,
+        "mistral-large-latest": 0.5154639175,
+        "mistral-medium-latest": 0.4719101124,
+        "llama 3 70b-instruct": 0.4020618557,
+        "mistral-small-latest": 0.6736842105
+    },
+    "Reflected Attack": {
+        "codellama-13b-instruct": 0.0602409639,
+        "codellama-34b-instruct": 0.1785714286,
+        "gpt-4": 0.1460674157,
+        "gpt-3.5-turbo": 0.0714285714,
+        "codellama-70b-instruct": 0.1704545455,
+        "llama 3 8b-instruct": 0.3469387755,
+        "mistral-large-latest": 0.3548387097,
+        "mistral-medium-latest": 0.4565217391,
+        "llama 3 70b-instruct": 0.2323232323,
+        "mistral-small-latest": 0.4
+    },
+    "Social Engineering": {
+        "codellama-13b-instruct": 0.0963855422,
+        "codellama-34b-instruct": 0.1290322581,
+        "gpt-4": 0.2073170732,
+        "gpt-3.5-turbo": 0.2359550562,
+        "codellama-70b-instruct": 0.2117647059,
+        "llama 3 8b-instruct": 0.3052631579,
+        "mistral-large-latest": 0.2421052632,
+        "mistral-medium-latest": 0.2150537634,
+        "llama 3 70b-instruct": 0.1770833333,
+        "mistral-small-latest": 0.3214285714
+    }
+}

prompt_injection_tests.json

@@ -0,0 +1,137 @@
+{
+    "different_user_input_language": {
+        "codellama-70b-instruct": 0.28,
+        "gpt-4": 0.16,
+        "llama 3 70b-instruct": 0.44,
+        "codellama-34b-instruct": 0.64,
+        "codellama-13b-instruct": 0.76,
+        "gpt-3.5-turbo": 0.6,
+        "llama 3 8b-instruct": 0.76
+    },
+    "output_formatting_manipulation": {
+        "codellama-70b-instruct": 0.1764705882,
+        "gpt-4": 0.3529411765,
+        "llama 3 70b-instruct": 0.7058823529,
+        "codellama-34b-instruct": 0.2941176471,
+        "codellama-13b-instruct": 0.2941176471,
+        "gpt-3.5-turbo": 0.7647058824,
+        "llama 3 8b-instruct": 0.7647058824
+    },
+    "overload_with_information": {
+        "codellama-70b-instruct": 0.2,
+        "gpt-4": 0.35,
+        "llama 3 70b-instruct": 0.25,
+        "codellama-34b-instruct": 0.6,
+        "codellama-13b-instruct": 0.55,
+        "gpt-3.5-turbo": 0.4,
+        "llama 3 8b-instruct": 0.6
+    },
+    "many_shot_attack": {
+        "codellama-70b-instruct": 0.0,
+        "gpt-4": 0.2857142857,
+        "llama 3 70b-instruct": 0.2857142857,
+        "codellama-34b-instruct": 0.7142857143,
+        "codellama-13b-instruct": 0.5714285714,
+        "gpt-3.5-turbo": 0.5714285714,
+        "llama 3 8b-instruct": 0.1428571429
+    },
+    "ignore_previous_instructions": {
+        "codellama-70b-instruct": 0.3333333333,
+        "gpt-4": 0.2083333333,
+        "llama 3 70b-instruct": 0.375,
+        "codellama-34b-instruct": 0.4583333333,
+        "codellama-13b-instruct": 0.2916666667,
+        "gpt-3.5-turbo": 0.3333333333,
+        "llama 3 8b-instruct": 0.5416666667
+    },
+    "system_mode": {
+        "codellama-70b-instruct": 0.1578947368,
+        "gpt-4": 0.1578947368,
+        "llama 3 70b-instruct": 0.4210526316,
+        "codellama-34b-instruct": 0.3684210526,
+        "codellama-13b-instruct": 0.5263157895,
+        "gpt-3.5-turbo": 0.3684210526,
+        "llama 3 8b-instruct": 0.5263157895
+    },
+    "few_shot_attack": {
+        "codellama-70b-instruct": 0.0,
+        "gpt-4": 0.1818181818,
+        "llama 3 70b-instruct": 0.1818181818,
+        "codellama-34b-instruct": 0.4545454545,
+        "codellama-13b-instruct": 0.5454545455,
+        "gpt-3.5-turbo": 0.3636363636,
+        "llama 3 8b-instruct": 0.6363636364
+    },
+    "indirect_reference": {
+        "codellama-70b-instruct": 0.2142857143,
+        "gpt-4": 0.4285714286,
+        "llama 3 70b-instruct": 0.3571428571,
+        "codellama-34b-instruct": 0.3571428571,
+        "codellama-13b-instruct": 0.2142857143,
+        "gpt-3.5-turbo": 0.3571428571,
+        "llama 3 8b-instruct": 0.3571428571
+    },
+    "repeated_token_attack": {
+        "codellama-70b-instruct": 0.0,
+        "gpt-4": 0.0,
+        "llama 3 70b-instruct": 0.0,
+        "codellama-34b-instruct": 0.5,
+        "codellama-13b-instruct": 0.5,
+        "gpt-3.5-turbo": 0.6666666667,
+        "llama 3 8b-instruct": 0.5
+    },
+    "persuasion": {
+        "codellama-70b-instruct": 0.3461538462,
+        "gpt-4": 0.1538461538,
+        "llama 3 70b-instruct": 0.2692307692,
+        "codellama-34b-instruct": 0.3076923077,
+        "codellama-13b-instruct": 0.2692307692,
+        "gpt-3.5-turbo": 0.1923076923,
+        "llama 3 8b-instruct": 0.4615384615
+    },
+    "mixed_techniques": {
+        "codellama-70b-instruct": 0.1515151515,
+        "gpt-4": 0.3333333333,
+        "llama 3 70b-instruct": 0.2424242424,
+        "codellama-34b-instruct": 0.2121212121,
+        "codellama-13b-instruct": 0.2727272727,
+        "gpt-3.5-turbo": 0.3636363636,
+        "llama 3 8b-instruct": 0.3636363636
+    },
+    "virtualization": {
+        "codellama-70b-instruct": 0.0,
+        "gpt-4": 0.1428571429,
+        "llama 3 70b-instruct": 0.4285714286,
+        "codellama-34b-instruct": 0.0714285714,
+        "codellama-13b-instruct": 0.2142857143,
+        "gpt-3.5-turbo": 0.2857142857,
+        "llama 3 8b-instruct": 0.5
+    },
+    "payload_splitting": {
+        "codellama-70b-instruct": 0.0,
+        "gpt-4": 0.0,
+        "llama 3 70b-instruct": 0.1111111111,
+        "codellama-34b-instruct": 0.3333333333,
+        "codellama-13b-instruct": 0.4444444444,
+        "gpt-3.5-turbo": 0.2222222222,
+        "llama 3 8b-instruct": 0.3333333333
+    },
+    "hypothetical_scenario": {
+        "codellama-70b-instruct": 0.0769230769,
+        "gpt-4": 0.2307692308,
+        "llama 3 70b-instruct": 0.2307692308,
+        "codellama-34b-instruct": 0.1538461538,
+        "codellama-13b-instruct": 0.1538461538,
+        "gpt-3.5-turbo": 0.3076923077,
+        "llama 3 8b-instruct": 0.2307692308
+    },
+    "token_smuggling": {
+        "codellama-70b-instruct": 0.0,
+        "gpt-4": 0.0,
+        "llama 3 70b-instruct": 0.0769230769,
+        "codellama-34b-instruct": 0.0,
+        "codellama-13b-instruct": 0.0,
+        "gpt-3.5-turbo": 0.0769230769,
+        "llama 3 8b-instruct": 0.0769230769
+    }
+}

trr_frr_tradeoff_helpfulness.json

@@ -0,0 +1,42 @@
+{
+    "codellama-13b-instruct": {
+        "Violation Rate": 0.45415284393545263,
+        "Refusal Rate": 0.016
+    },
+    "codellama-34b-instruct": {
+        "Violation Rate": 0.4996666666666666,
+        "Refusal Rate": 0.016
+    },
+    "codellama/CodeLlama-70b-Instruct-hf": {
+        "Violation Rate": 0.36645819397993296,
+        "Refusal Rate": 0.6946666666666667
+    },
+    "gpt-3.5-turbo": {
+        "Violation Rate": 0.6056666666666667,
+        "Refusal Rate": 0.0026666666666666666
+    },
+    "gpt-4": {
+        "Violation Rate": 0.28800000000000003,
+        "Refusal Rate": 0.04133333333333333
+    },
+    "mistral-medium-latest": {
+        "Violation Rate": 0.3470992196209587,
+        "Refusal Rate": 0.11333333333333333
+    },
+    "mistral-small-latest": {
+        "Violation Rate": 0.5693333333333334,
+        "Refusal Rate": 0.104
+    },
+    "mistral-large-latest": {
+        "Violation Rate": 0.4773333333333334,
+        "Refusal Rate": 0.11066666666666666
+    },
+    "llama3p-70b-rc3_vr_mid_3": {
+        "Violation Rate": 0.3663333333333333,
+        "Refusal Rate": 0.0
+    },
+    "llama3p-7b-rc3_vr_mid_2": {
+        "Violation Rate": 0.16366666666666663,
+        "Refusal Rate": 0.02266666666666667
+    }
+}