update README.md

app.py

@@ -83,8 +83,8 @@ def build_leaderboard(
 
     gr.Markdown(TABLE_INTRODUCTION, elem_classes="markdown-text")
     data_spilt_radio = gr.Radio(
-        choices=["Chinses", "English"],
-        value="Chinses",
+        choices=["Chinese", "English"],
+        value="Chinese",
         label=SELECT_SET_INTRO,
     )
 

constants.py

@@ -32,11 +32,13 @@ XLSX_DIR = "./file//results.xlsx"
 
 LEADERBOARD_INTRODUCTION = """# 🏆 S-Eval Leaderboard 
     ## 🔔 Updates
+    📣 [2024/05/31]: We release 20,000 corresponding attack prompts.
+
     📣 [2024/05/23]: We publish our [paper](https://arxiv.org/abs/2405.14191) and first release 2,000 base risk prompts.
 
     ### ❗️ Note
     Due to the limited machine resource, please refresh the page if a connection timeout error occurs.
-    
+
     You can get more detailed information from our [Project](https://github.com/IS2Lab/S-Eval) and [Paper](https://arxiv.org/abs/2405.14191).
 """
 
@@ -45,11 +47,12 @@ SELECT_SET_INTRO = (
 )
 
 TABLE_INTRODUCTION_1 = """In the table below, we summarize the safety scores  (%) of differnet models on Base Risk Prompt Set."""
-TABLE_INTRODUCTION_2 = """In the table below, we summarize the attack success rate (%) of the instruction attacks in Attack Prompt Set on different models"""
+TABLE_INTRODUCTION_2 = """In the table below, we summarize the attack success rates (%) of the instruction attacks in Attack Prompt Set on different models"""
 
 
 LEADERBORAD_INFO = """
-    S-Eval is a new comprehensive, multi-dimensional and open-ended safety evaluation benchmark. It consists of 220,000 evaluation prompts, including 20,000 base risk prompts (10,000 in Chinese and 10,000 in English) and 200, 000 corresponding attack prompts derived from 10 popular adversarial instruction attacks. These test prompts are generated based on a comprehensive and unified risk taxonomy. The risk taxonomy has a structured hierarchy with four levels, comprising 8 risk dimensions, 25 risk categories, 56 risk subcategories, and 52 risk sub-subcategories, specifically designed to encompass all crucial dimensions of safety evaluation and accurately reflect the varied safety levels of LLMs across different risk dimensions.
+   S-Eval is designed to be a new comprehensive, multi-dimensional and open-ended safety evaluation benchmark. So far, S-Eval has 220,000 evaluation prompts in total (and is still in active expansion), including 20,000 base risk prompts (10,000 in Chinese and 10,000 in English) and 200,000 *corresponding* attack prompts derived from 10 popular adversarial instruction attacks. These test prompts are generated based on a comprehensive and unified risk taxonomy, specifically designed to encompass all crucial dimensions of LLM safety evaluation and meant to accurately reflect the varied safety levels of LLMs across these risk dimensions. 
+More details on the construction of the test suite including model-based test generation, selection and the expert critique LLM can be found in our [paper](https://arxiv.org/abs/2405.14191). 
 """
 
 

src/utils_display.py

@@ -1,121 +1,143 @@
-from dataclasses import dataclass
+# from dataclasses import dataclass
 
 
 # These classes are for user facing column names, to avoid having to change them
 # all around the code when a modif is needed
-@dataclass
-class ColumnContent:
-    name: str
-    type: str
-    displayed_by_default: bool
-    hidden: bool = False
-    never_hidden: bool = False
-    dummy: bool = False
-
-
-def fields(raw_class):
-    return [
-        v for k, v in raw_class.__dict__.items() if k[:2] != "__" and k[-2:] != "__"
-    ]
-
-
-@dataclass(frozen=True)
-class AutoEvalColumn:  # Auto evals column
-
-    model_type_symbol = ColumnContent("T", "str", True)
-    model = ColumnContent("Model", "markdown", True, never_hidden=True)
-    average = ColumnContent("Average ⬆️", "number", True)
-    arc = ColumnContent("ARC", "number", True)
-    hellaswag = ColumnContent("HellaSwag", "number", True)
-    mmlu = ColumnContent("MMLU", "number", True)
-    truthfulqa = ColumnContent("TruthfulQA", "number", True)
-    model_type = ColumnContent("Type", "str", False)
-    precision = ColumnContent("Precision", "str", False, True)
-    license = ColumnContent("Hub License", "str", False)
-    params = ColumnContent("#Params (B)", "number", False)
-    likes = ColumnContent("Hub ❤️", "number", False)
-    revision = ColumnContent("Model sha", "str", False, False)
-    dummy = ColumnContent(
-        "model_name_for_query", "str", True
-    )  # dummy col to implement search bar (hidden by custom CSS)
-
-
-@dataclass(frozen=True)
-class EloEvalColumn:  # Elo evals column
-    model = ColumnContent("Model", "markdown", True)
-    gpt4 = ColumnContent("GPT-4 (all)", "number", True)
-    human_all = ColumnContent("Human (all)", "number", True)
-    human_instruct = ColumnContent("Human (instruct)", "number", True)
-    human_code_instruct = ColumnContent("Human (code-instruct)", "number", True)
-
-
-@dataclass(frozen=True)
-class EvalQueueColumn:  # Queue column
-    model = ColumnContent("model", "markdown", True)
-    revision = ColumnContent("revision", "str", True)
-    private = ColumnContent("private", "bool", True)
-    precision = ColumnContent("precision", "bool", True)
-    weight_type = ColumnContent("weight_type", "str", "Original")
-    status = ColumnContent("status", "str", True)
-
-
-LLAMAS = [
-    "huggingface/llama-7b",
-    "huggingface/llama-13b",
-    "huggingface/llama-30b",
-    "huggingface/llama-65b",
-]
-
-
-KOALA_LINK = "https://huggingface.co/TheBloke/koala-13B-HF"
-VICUNA_LINK = "https://huggingface.co/lmsys/vicuna-13b-delta-v1.1"
-OASST_LINK = "https://huggingface.co/OpenAssistant/oasst-sft-4-pythia-12b-epoch-3.5"
-DOLLY_LINK = "https://huggingface.co/databricks/dolly-v2-12b"
-MODEL_PAGE = "https://huggingface.co/models"
-LLAMA_LINK = "https://ai.facebook.com/blog/large-language-model-llama-meta-ai/"
-VICUNA_LINK = "https://huggingface.co/CarperAI/stable-vicuna-13b-delta"
-ALPACA_LINK = "https://crfm.stanford.edu/2023/03/13/alpaca.html"
-
-
-def model_hyperlink(link, model_name):
-    return f'<a target="_blank" href="{link}" style="color: var(--link-text-color); text-decoration: underline;text-decoration-style: dotted;">{model_name}</a>'
-
-
-def make_clickable_model(model_name):
-    link = f"https://huggingface.co/{model_name}"
-
-    if model_name in LLAMAS:
-        link = LLAMA_LINK
-        model_name = model_name.split("/")[1]
-    elif model_name == "HuggingFaceH4/stable-vicuna-13b-2904":
-        link = VICUNA_LINK
-        model_name = "stable-vicuna-13b"
-    elif model_name == "HuggingFaceH4/llama-7b-ift-alpaca":
-        link = ALPACA_LINK
-        model_name = "alpaca-13b"
-    if model_name == "dolly-12b":
-        link = DOLLY_LINK
-    elif model_name == "vicuna-13b":
-        link = VICUNA_LINK
-    elif model_name == "koala-13b":
-        link = KOALA_LINK
-    elif model_name == "oasst-12b":
-        link = OASST_LINK
+# @dataclass
+# class ColumnContent:
+#     name: str
+#     type: str
+#     displayed_by_default: bool
+#     hidden: bool = False
+#     never_hidden: bool = False
+#     dummy: bool = False
+
+
+# def fields(raw_class):
+#     return [
+#         v for k, v in raw_class.__dict__.items() if k[:2] != "__" and k[-2:] != "__"
+#     ]
+
+
+# @dataclass(frozen=True)
+# class AutoEvalColumn:  # Auto evals column
+
+#     model_type_symbol = ColumnContent("T", "str", True)
+#     model = ColumnContent("Model", "markdown", True, never_hidden=True)
+#     average = ColumnContent("Average ⬆️", "number", True)
+#     arc = ColumnContent("ARC", "number", True)
+#     hellaswag = ColumnContent("HellaSwag", "number", True)
+#     mmlu = ColumnContent("MMLU", "number", True)
+#     truthfulqa = ColumnContent("TruthfulQA", "number", True)
+#     model_type = ColumnContent("Type", "str", False)
+#     precision = ColumnContent("Precision", "str", False, True)
+#     license = ColumnContent("Hub License", "str", False)
+#     params = ColumnContent("#Params (B)", "number", False)
+#     likes = ColumnContent("Hub ❤️", "number", False)
+#     revision = ColumnContent("Model sha", "str", False, False)
+#     dummy = ColumnContent(
+#         "model_name_for_query", "str", True
+#     )  # dummy col to implement search bar (hidden by custom CSS)
+
+
+# @dataclass(frozen=True)
+# class EloEvalColumn:  # Elo evals column
+#     model = ColumnContent("Model", "markdown", True)
+#     gpt4 = ColumnContent("GPT-4 (all)", "number", True)
+#     human_all = ColumnContent("Human (all)", "number", True)
+#     human_instruct = ColumnContent("Human (instruct)", "number", True)
+#     human_code_instruct = ColumnContent("Human (code-instruct)", "number", True)
+
+
+# @dataclass(frozen=True)
+# class EvalQueueColumn:  # Queue column
+#     model = ColumnContent("model", "markdown", True)
+#     revision = ColumnContent("revision", "str", True)
+#     private = ColumnContent("private", "bool", True)
+#     precision = ColumnContent("precision", "bool", True)
+#     weight_type = ColumnContent("weight_type", "str", "Original")
+#     status = ColumnContent("status", "str", True)
+
+
+# LLAMAS = [
+#     "huggingface/llama-7b",
+#     "huggingface/llama-13b",
+#     "huggingface/llama-30b",
+#     "huggingface/llama-65b",
+# ]
+
+
+# KOALA_LINK = "https://huggingface.co/TheBloke/koala-13B-HF"
+# VICUNA_LINK = "https://huggingface.co/lmsys/vicuna-13b-delta-v1.1"
+# OASST_LINK = "https://huggingface.co/OpenAssistant/oasst-sft-4-pythia-12b-epoch-3.5"
+# DOLLY_LINK = "https://huggingface.co/databricks/dolly-v2-12b"
+# MODEL_PAGE = "https://huggingface.co/models"
+# LLAMA_LINK = "https://ai.facebook.com/blog/large-language-model-llama-meta-ai/"
+# VICUNA_LINK = "https://huggingface.co/CarperAI/stable-vicuna-13b-delta"
+# ALPACA_LINK = "https://crfm.stanford.edu/2023/03/13/alpaca.html"
+
+
+# def model_hyperlink(link, model_name):
+#     return f'<a target="_blank" href="{link}" style="color: var(--link-text-color); text-decoration: underline;text-decoration-style: dotted;">{model_name}</a>'
+
+
+# def make_clickable_model(model_name):
+#     link = f"https://huggingface.co/{model_name}"
+
+#     if model_name in LLAMAS:
+#         link = LLAMA_LINK
+#         model_name = model_name.split("/")[1]
+#     elif model_name == "HuggingFaceH4/stable-vicuna-13b-2904":
+#         link = VICUNA_LINK
+#         model_name = "stable-vicuna-13b"
+#     elif model_name == "HuggingFaceH4/llama-7b-ift-alpaca":
+#         link = ALPACA_LINK
+#         model_name = "alpaca-13b"
+#     if model_name == "dolly-12b":
+#         link = DOLLY_LINK
+#     elif model_name == "vicuna-13b":
+#         link = VICUNA_LINK
+#     elif model_name == "koala-13b":
+#         link = KOALA_LINK
+#     elif model_name == "oasst-12b":
+#         link = OASST_LINK
     # else:
     #    link = MODEL_PAGE
 
-    return model_hyperlink(link, model_name)
-
-
-def styled_error(error):
-    return f"<p style='color: red; font-size: 20px; text-align: center;'>{error}</p>"
-
-
-def styled_warning(warn):
-    return f"<p style='color: orange; font-size: 20px; text-align: center;'>{warn}</p>"
-
-
-def styled_message(message):
-    return (
-        f"<p style='color: green; font-size: 20px; text-align: center;'>{message}</p>"
-    )
+#     return model_hyperlink(link, model_name)
+
+
+# def styled_error(error):
+#     return f"<p style='color: red; font-size: 20px; text-align: center;'>{error}</p>"
+
+
+# def styled_warning(warn):
+#     return f"<p style='color: orange; font-size: 20px; text-align: center;'>{warn}</p>"
+
+
+# def styled_message(message):
+#     return (
+#         f"<p style='color: green; font-size: 20px; text-align: center;'>{message}</p>"
+#     )
+
+Qwen_1_8B_Chat_Link = "https://huggingface.co/Qwen/Qwen-1_8B-Chat"
+Qwen_7B_Chat_Link = "https://huggingface.co/Qwen/Qwen-7B-Chat"
+Qwen_14B_Chat_Link = "https://huggingface.co/Qwen/Qwen-14B-Chat"
+Qwen_72B_Chat_Link = "https://huggingface.co/Qwen/Qwen-72B-Chat"
+Gemma_2B_it_Link = "https://huggingface.co/google/gemma-2b-it"
+Gemma_7B_it__Link = "https://huggingface.co/google/gemma-7b-it"
+ChatGLM3_6B_Link = "https://huggingface.co/THUDM/chatglm3-6b"
+Mistral_7B_Instruct_v0_2_Link = "https://huggingface.co/mistralai/Mistral-7B-Instruct-v0.2"
+LLaMA_2_7B_Chat_Link = "https://huggingface.co/meta-llama/Llama-2-7b-chat-hf"
+LLaMA_2_13B_Chat_Link = "https://huggingface.co/meta-llama/Llama-2-13b-chat-hf"
+LLaMA_2_70B_Chat_Link = "https://huggingface.co/meta-llama/Llama-2-70b-chat-hf"
+LLaMA_3_8B_Instruct_Link = "https://huggingface.co/meta-llama/Meta-Llama-3-8B-Instruct"
+LLaMA_3_70B_Instruct_Link = "https://huggingface.co/meta-llama/Meta-Llama-3-70B-Instruct"
+Vicuna_7B_v1_3_Link = "https://huggingface.co/lmsys/vicuna-7b-v1.3"
+Vicuna_13B_v1_3_Link = "https://huggingface.co/lmsys/vicuna-13b-v1.3"
+Vicuna_33B_v1_3_Link = "https://huggingface.co/lmsys/vicuna-33b-v1.3"
+Baichuan2_13B_Chat_Link = "https://huggingface.co/baichuan-inc/Baichuan2-13B-Chat"
+Yi_34B_Chat_Link = "https://huggingface.co/01-ai/Yi-34B-Chat"
+GPT_4_Turbo_Link = "https://platform.openai.com/docs/models/gpt-4-turbo-and-gpt-4"
+ErnieBot_4_0_Link = "https://cloud.baidu.com/doc/WENXINWORKSHOP/s/clntwmv7t"
+Gemini_1_0_Pro_Link = "https://ai.google.dev/gemini-api/docs/models/gemini"