Update_S-Eval_v0.1.2

README.md

@@ -1,13 +1,45 @@
 ---
-title: S-Eval V0.1.2
-emoji: 😻
-colorFrom: blue
-colorTo: purple
+title: 🏆 S-Eval Leaderboard
+emoji: 🥇
+colorFrom: green
+colorTo: indigo
 sdk: gradio
 sdk_version: 5.3.0
 app_file: app.py
-pinned: false
+pinned: true
 license: cc-by-nc-sa-4.0
 ---
 
-Check out the configuration reference at https://huggingface.co/docs/hub/spaces-config-reference
+# Start the configuration
+
+Most of the variables to change for a default leaderboard are in `src/env.py` (replace the path for your leaderboard) and `src/about.py` (for tasks).
+
+Results files should have the following format and be stored as json files:
+```json
+{
+    "config": {
+        "model_dtype": "torch.float16", # or torch.bfloat16 or 8bit or 4bit
+        "model_name": "path of the model on the hub: org/model",
+        "model_sha": "revision on the hub",
+    },
+    "results": {
+        "task_name": {
+            "metric_name": score,
+        },
+        "task_name2": {
+            "metric_name": score,
+        }
+    }
+}
+```
+
+Request files are created automatically by this tool.
+
+If you encounter problem on the space, don't hesitate to restart it to remove the create eval-queue, eval-queue-bk, eval-results and eval-results-bk created folder.
+
+# Code logic for more complex edits
+
+You'll find 
+- the main table' columns names and properties in `src/display/utils.py`
+- the logic to read all results and request files, then convert them in dataframe lines, in `src/leaderboard/read_evals.py`, and `src/populate.py`
+- teh logic to allow or filter submissions in `src/submission/submit.py` and `src/submission/check_validity.py`

app.py

@@ -0,0 +1,178 @@
+import gradio as gr
+
+__all__ = ["block", "make_clickable_model", "make_clickable_user", "get_submissions"]
+
+
+import numpy as np
+import pandas as pd
+
+from constants import *
+from src.auto_leaderboard.model_metadata_type import ModelType
+
+global data_component, filter_component, ref_dic
+
+
+def upload_file(files):
+    file_paths = [file.name for file in files]
+    return file_paths
+
+
+def read_xlsx_leaderboard():
+    df_dict = pd.read_excel(XLSX_DIR, sheet_name=None)  # get all sheet
+    return df_dict
+
+
+def get_specific_df(sheet_name):
+    df = read_xlsx_leaderboard()[sheet_name].sort_values(by="Overall", ascending=False)
+    return df
+
+def get_link_df(sheet_name):
+    df = read_xlsx_leaderboard()[sheet_name]
+    return df
+
+
+ref_df = get_link_df("main")
+
+ref_dic = {}
+for id, row in ref_df.iterrows():
+    
+    ref_dic[
+        str(row["Model"])
+    ] = f'<a href="{row["Link"]}" target="_blank"  style="color: var(--link-text-color); text-decoration: underline;text-decoration-style: dotted;">{row["Model"]}</a>'
+
+def wrap_model(func):
+    def wrapper(*args, **kwargs):
+        df = func(*args, **kwargs)
+        df["Model"] = df["Model"].apply(lambda x: ref_dic[x])
+        # cols_to_round = df.select_dtypes(include=[np.number]).columns.tolist()
+        # cols_to_round = [col for col in cols_to_round if col != "Model"]
+        # df[cols_to_round] = df[cols_to_round].apply(lambda x: np.round(x, 2))
+
+        all_cols = df.columns.tolist()
+        non_numeric_cols = df.select_dtypes(exclude=[np.number]).columns.tolist()
+        cols_to_round = [col for col in all_cols if col not in non_numeric_cols and col != "Model"]
+        df[cols_to_round] = df[cols_to_round].apply(lambda x: np.round(x, 2))
+        return df
+
+    return wrapper
+
+
+@wrap_model
+def get_base_zh_df():
+    return get_specific_df("base-zh")
+
+
+@wrap_model
+def get_base_en_df():
+    return get_specific_df("base-en")
+
+
+@wrap_model
+def get_attack_zh_df():
+    return get_specific_df("attack-zh")
+
+
+@wrap_model
+def get_attack_en_df():
+    return get_specific_df("attack-en")
+
+
+def build_leaderboard(
+    TABLE_INTRODUCTION, TAX_COLUMNS, get_chinese_df, get_english_df
+):
+
+    gr.Markdown(TABLE_INTRODUCTION, elem_classes="markdown-text")
+    data_spilt_radio = gr.Radio(
+        choices=["Chinese", "English"],
+        value="Chinese",
+        label=SELECT_SET_INTRO,
+    )
+
+    # 创建数据帧组件
+    data_component = gr.components.Dataframe(
+        value=get_chinese_df,
+        headers=OVERALL_INFO + TAX_COLUMNS,
+        type="pandas",
+        datatype=["markdown"] + ["number"] + ["number"] * len(TAX_COLUMNS),
+        interactive=False,
+        visible=True,
+        wrap=True,
+        column_widths=[250] + [100] + [150] * len(TAX_COLUMNS),
+    )
+
+    def on_data_split_radio(seleted_split):
+        if "Chinese" in seleted_split:
+            updated_data = get_chinese_df()
+        if "English" in seleted_split:
+            updated_data = get_english_df()
+        current_columns = data_component.headers  # 获取的当前的column
+        current_datatype = data_component.datatype  # 获取当前的datatype
+        filter_component = gr.components.Dataframe(
+            value=updated_data,
+            headers=current_columns,
+            type="pandas",
+            datatype=current_datatype,
+            interactive=False,
+            visible=True,
+            wrap=True,
+            column_widths=[250] + [100] + [150] * (len(current_columns) - 2),
+        )
+        return filter_component
+
+    # 关联处理函数
+    data_spilt_radio.change(
+        fn=on_data_split_radio, inputs=data_spilt_radio, outputs=data_component
+    )
+
+
+def build_demo():
+    block = gr.Blocks()
+
+    with block:
+        gr.Markdown(LEADERBOARD_INTRODUCTION)
+
+        with gr.Tabs(elem_classes="tab-buttons") as tabs:
+            # first
+            with gr.TabItem(
+                "Base Risk Prompt Set Results",
+                elem_id="evalcrafter-benchmark-tab-table",
+                id=0,
+            ):
+                build_leaderboard(
+                    TABLE_INTRODUCTION_1,
+                    risk_topic_1_columns,
+                    get_base_zh_df,
+                    get_base_en_df
+                )
+            # second
+            with gr.TabItem(
+                "Attack Prompt Set Results",
+                elem_id="evalcrafter-benchmark-tab-table",
+                id=1,
+            ):
+                build_leaderboard(
+                    TABLE_INTRODUCTION_2,
+                    attack_columns,
+                    get_attack_zh_df,
+                    get_attack_en_df
+                )
+            # last table about
+            with gr.TabItem("📝 About", elem_id="evalcrafter-benchmark-tab-table", id=3):
+                gr.Markdown(LEADERBORAD_INFO, elem_classes="markdown-text")
+
+        with gr.Row():
+            with gr.Accordion("📙 Citation", open=True):
+                citation_button = gr.Textbox(
+                    value=CITATION_BUTTON_TEXT,
+                    label=CITATION_BUTTON_LABEL,
+                    lines=10,
+                    elem_id="citation-button",
+                    show_label=True,
+                    show_copy_button=True,
+                )
+
+    # block.launch(share=True)
+    block.launch()
+
+if __name__ == "__main__":
+    build_demo()

constants.py

@@ -0,0 +1,73 @@
+# constants
+OVERALL_INFO = ["Model", "Overall"]
+
+risk_topic_1_columns = [
+    "Crimes and Illegal Activities",
+    "Cybersecurity",
+    "Data Privacy",
+    "Ethics and Morality",
+    "Physical and Mental Health",
+    "Hate Speech",
+    "Extremism",
+    "Inappropriate Suggestions"
+]
+risk_topic_1_columns = [item.lower() for item in risk_topic_1_columns]
+
+attack_columns = [
+    "Adaptive Attack",
+    "Positive Induction",
+    "Reverse Induction",
+    "Code Injection",
+    "Instruction Jailbreak",
+    "Goal Hijacking",
+    "Instruction Encryption",
+    "DeepInception",
+    "In-Context Attack",
+    "Chain of Utterances",
+    "Compositional Instructions"
+]
+attack_columns = [item.lower() for item in attack_columns]
+
+XLSX_DIR = "./file//results.xlsx"
+
+LEADERBOARD_INTRODUCTION = """# 🏆 S-Eval Leaderboard 
+    ## 🔔 Updates
+    📣 [2024/10/25]: We release all 20,000 base risk prompts and 200,000 corresponding attack prompts ([Version-0.1.2](https://github.com/IS2Lab/S-Eval)). We also update [🏆 LeaderBoard v0.1.2](https://huggingface.co/spaces/IS2Lab/S-Eval_v0.1.2) with new evaluation results including GPT-4 and other models.  
+    🎉 S-Eval has about 7,000 total views and about 2,000 total downloads across multiple platforms 🎉.
+
+    📣 [2024/06/17]: We further release 10,000 base risk prompts and 100,000 corresponding attack prompts ([Version-0.1.1](https://github.com/IS2Lab/S-Eval)). If you require automatic safety evaluations, please feel free to submit a request via [Issues](https://huggingface.co/spaces/IS2Lab/S-Eval/discussions) or contact us by [Email](mailto:xiaohanyuan@zju.edu.cn).
+
+    📣 [2024/05/31]: We release 20,000 corresponding attack prompts.
+
+    📣 [2024/05/23]: We publish our [paper](https://arxiv.org/abs/2405.14191) and first release 2,000 base risk prompts. You can download the benchmark from our [project](https://github.com/IS2Lab/S-Eval), the [HuggingFace Dataset](https://huggingface.co/datasets/IS2Lab/S-Eval).
+
+    ### ❗️ Note
+    Due to the limited machine resource, please refresh the page if a connection timeout error occurs.
+
+    You can get more detailed information from our [Project](https://github.com/IS2Lab/S-Eval) and [Paper](https://arxiv.org/abs/2405.14191).
+"""
+
+SELECT_SET_INTRO = (
+    "Select whether Chinese or English results should be shown."
+)
+
+TABLE_INTRODUCTION_1 = """In the table below, we summarize the safety scores  (%) of differnet models on Base Risk Prompt Set."""
+TABLE_INTRODUCTION_2 = """In the table below, we summarize the attack success rates (%) of the instruction attacks in Attack Prompt Set on different models"""
+
+
+LEADERBORAD_INFO = """
+   S-Eval is designed to be a new comprehensive, multi-dimensional and open-ended safety evaluation benchmark. So far, S-Eval has 220,000 evaluation prompts in total (and is still in active expansion), including 20,000 base risk prompts (10,000 in Chinese and 10,000 in English) and 200,000 *corresponding* attack prompts derived from 10 popular adversarial instruction attacks. These test prompts are generated based on a comprehensive and unified risk taxonomy, specifically designed to encompass all crucial dimensions of LLM safety evaluation and meant to accurately reflect the varied safety levels of LLMs across these risk dimensions. 
+More details on the construction of the test suite including model-based test generation, selection and the expert critique LLM can be found in our [paper](https://arxiv.org/abs/2405.14191). 
+"""
+
+
+CITATION_BUTTON_LABEL = "If our work is useful for your own, you can cite us with the following BibTex entry:"
+
+CITATION_BUTTON_TEXT = r"""
+@article{yuan2024seval,
+  title={S-Eval: Automatic and Adaptive Test Generation for Benchmarking Safety Evaluation of Large Language Models},
+  author={Xiaohan Yuan and Jinfeng Li and Dongxia Wang and Yuefeng Chen and Xiaofeng Mao and Longtao Huang and Hui Xue and Wenhai Wang and Kui Ren and Jingyi Wang},
+  journal={arXiv preprint arXiv:2405.14191},
+  year={2024}
+}
+"""

requirements.txt

@@ -0,0 +1,16 @@
+APScheduler==3.10.1
+black==23.11.0
+click==8.1.3
+datasets==2.14.5
+gradio
+gradio_client
+huggingface-hub>=0.18.0
+matplotlib==3.7.1
+numpy==1.24.2
+openpyxl==3.1.2
+pandas==2.0.0
+plotly==5.14.1
+python-dateutil==2.8.2
+requests==2.28.2
+sentencepiece
+tqdm==4.65.0

src/auto_leaderboard/model_metadata_type.py

@@ -0,0 +1,26 @@
+from dataclasses import dataclass
+from enum import Enum
+
+
+@dataclass
+class ModelInfo:
+    name: str
+    symbol: str  # emoji
+
+
+model_type_symbols = {
+    "LLM": "🟢",
+    "ImageLLM": "🔶",
+    "VideoLLM": "⭕",
+    "Other": "🟦",
+}
+
+
+class ModelType(Enum):
+    PT = ModelInfo(name="LLM", symbol="🟢")
+    FT = ModelInfo(name="ImageLLM", symbol="🔶")
+    IFT = ModelInfo(name="VideoLLM", symbol="⭕")
+    RL = ModelInfo(name="Other", symbol="🟦")
+
+    def to_str(self, separator=" "):
+        return f"{self.value.symbol}{separator}{self.value.name}"

src/utils_display.py

@@ -0,0 +1,143 @@
+# from dataclasses import dataclass
+
+
+# These classes are for user facing column names, to avoid having to change them
+# all around the code when a modif is needed
+# @dataclass
+# class ColumnContent:
+#     name: str
+#     type: str
+#     displayed_by_default: bool
+#     hidden: bool = False
+#     never_hidden: bool = False
+#     dummy: bool = False
+
+
+# def fields(raw_class):
+#     return [
+#         v for k, v in raw_class.__dict__.items() if k[:2] != "__" and k[-2:] != "__"
+#     ]
+
+
+# @dataclass(frozen=True)
+# class AutoEvalColumn:  # Auto evals column
+
+#     model_type_symbol = ColumnContent("T", "str", True)
+#     model = ColumnContent("Model", "markdown", True, never_hidden=True)
+#     average = ColumnContent("Average ⬆️", "number", True)
+#     arc = ColumnContent("ARC", "number", True)
+#     hellaswag = ColumnContent("HellaSwag", "number", True)
+#     mmlu = ColumnContent("MMLU", "number", True)
+#     truthfulqa = ColumnContent("TruthfulQA", "number", True)
+#     model_type = ColumnContent("Type", "str", False)
+#     precision = ColumnContent("Precision", "str", False, True)
+#     license = ColumnContent("Hub License", "str", False)
+#     params = ColumnContent("#Params (B)", "number", False)
+#     likes = ColumnContent("Hub ❤️", "number", False)
+#     revision = ColumnContent("Model sha", "str", False, False)
+#     dummy = ColumnContent(
+#         "model_name_for_query", "str", True
+#     )  # dummy col to implement search bar (hidden by custom CSS)
+
+
+# @dataclass(frozen=True)
+# class EloEvalColumn:  # Elo evals column
+#     model = ColumnContent("Model", "markdown", True)
+#     gpt4 = ColumnContent("GPT-4 (all)", "number", True)
+#     human_all = ColumnContent("Human (all)", "number", True)
+#     human_instruct = ColumnContent("Human (instruct)", "number", True)
+#     human_code_instruct = ColumnContent("Human (code-instruct)", "number", True)
+
+
+# @dataclass(frozen=True)
+# class EvalQueueColumn:  # Queue column
+#     model = ColumnContent("model", "markdown", True)
+#     revision = ColumnContent("revision", "str", True)
+#     private = ColumnContent("private", "bool", True)
+#     precision = ColumnContent("precision", "bool", True)
+#     weight_type = ColumnContent("weight_type", "str", "Original")
+#     status = ColumnContent("status", "str", True)
+
+
+# LLAMAS = [
+#     "huggingface/llama-7b",
+#     "huggingface/llama-13b",
+#     "huggingface/llama-30b",
+#     "huggingface/llama-65b",
+# ]
+
+
+# KOALA_LINK = "https://huggingface.co/TheBloke/koala-13B-HF"
+# VICUNA_LINK = "https://huggingface.co/lmsys/vicuna-13b-delta-v1.1"
+# OASST_LINK = "https://huggingface.co/OpenAssistant/oasst-sft-4-pythia-12b-epoch-3.5"
+# DOLLY_LINK = "https://huggingface.co/databricks/dolly-v2-12b"
+# MODEL_PAGE = "https://huggingface.co/models"
+# LLAMA_LINK = "https://ai.facebook.com/blog/large-language-model-llama-meta-ai/"
+# VICUNA_LINK = "https://huggingface.co/CarperAI/stable-vicuna-13b-delta"
+# ALPACA_LINK = "https://crfm.stanford.edu/2023/03/13/alpaca.html"
+
+
+# def model_hyperlink(link, model_name):
+#     return f'<a target="_blank" href="{link}" style="color: var(--link-text-color); text-decoration: underline;text-decoration-style: dotted;">{model_name}</a>'
+
+
+# def make_clickable_model(model_name):
+#     link = f"https://huggingface.co/{model_name}"
+
+#     if model_name in LLAMAS:
+#         link = LLAMA_LINK
+#         model_name = model_name.split("/")[1]
+#     elif model_name == "HuggingFaceH4/stable-vicuna-13b-2904":
+#         link = VICUNA_LINK
+#         model_name = "stable-vicuna-13b"
+#     elif model_name == "HuggingFaceH4/llama-7b-ift-alpaca":
+#         link = ALPACA_LINK
+#         model_name = "alpaca-13b"
+#     if model_name == "dolly-12b":
+#         link = DOLLY_LINK
+#     elif model_name == "vicuna-13b":
+#         link = VICUNA_LINK
+#     elif model_name == "koala-13b":
+#         link = KOALA_LINK
+#     elif model_name == "oasst-12b":
+#         link = OASST_LINK
+    # else:
+    #    link = MODEL_PAGE
+
+#     return model_hyperlink(link, model_name)
+
+
+# def styled_error(error):
+#     return f"<p style='color: red; font-size: 20px; text-align: center;'>{error}</p>"
+
+
+# def styled_warning(warn):
+#     return f"<p style='color: orange; font-size: 20px; text-align: center;'>{warn}</p>"
+
+
+# def styled_message(message):
+#     return (
+#         f"<p style='color: green; font-size: 20px; text-align: center;'>{message}</p>"
+#     )
+
+Qwen_1_8B_Chat_Link = "https://huggingface.co/Qwen/Qwen-1_8B-Chat"
+Qwen_7B_Chat_Link = "https://huggingface.co/Qwen/Qwen-7B-Chat"
+Qwen_14B_Chat_Link = "https://huggingface.co/Qwen/Qwen-14B-Chat"
+Qwen_72B_Chat_Link = "https://huggingface.co/Qwen/Qwen-72B-Chat"
+Gemma_2B_it_Link = "https://huggingface.co/google/gemma-2b-it"
+Gemma_7B_it__Link = "https://huggingface.co/google/gemma-7b-it"
+ChatGLM3_6B_Link = "https://huggingface.co/THUDM/chatglm3-6b"
+Mistral_7B_Instruct_v0_2_Link = "https://huggingface.co/mistralai/Mistral-7B-Instruct-v0.2"
+LLaMA_2_7B_Chat_Link = "https://huggingface.co/meta-llama/Llama-2-7b-chat-hf"
+LLaMA_2_13B_Chat_Link = "https://huggingface.co/meta-llama/Llama-2-13b-chat-hf"
+LLaMA_2_70B_Chat_Link = "https://huggingface.co/meta-llama/Llama-2-70b-chat-hf"
+LLaMA_3_8B_Instruct_Link = "https://huggingface.co/meta-llama/Meta-Llama-3-8B-Instruct"
+LLaMA_3_70B_Instruct_Link = "https://huggingface.co/meta-llama/Meta-Llama-3-70B-Instruct"
+Vicuna_7B_v1_3_Link = "https://huggingface.co/lmsys/vicuna-7b-v1.3"
+Vicuna_13B_v1_3_Link = "https://huggingface.co/lmsys/vicuna-13b-v1.3"
+Vicuna_33B_v1_3_Link = "https://huggingface.co/lmsys/vicuna-33b-v1.3"
+Baichuan2_13B_Chat_Link = "https://huggingface.co/baichuan-inc/Baichuan2-13B-Chat"
+Yi_34B_Chat_Link = "https://huggingface.co/01-ai/Yi-34B-Chat"
+GPT_4_Turbo_Link = "https://platform.openai.com/docs/models/gpt-4-turbo-and-gpt-4"
+ErnieBot_4_0_Link = "https://cloud.baidu.com/doc/WENXINWORKSHOP/s/clntwmv7t"
+Gemini_1_0_Pro_Link = "https://ai.google.dev/gemini-api/docs/models/gemini"