init

.gitignore

@@ -5,9 +5,10 @@ __pycache__/
 .ipynb_checkpoints
 *ipynb
 .vscode/
+.huggingface
 
-eval-queue/
-eval-results/
-eval-queue-bk/
-eval-results-bk/
+# eval-queue/
+# eval-results/
+# eval-queue-bk/
+# eval-results-bk/
 logs/

app.py

@@ -34,29 +34,13 @@ from src.submission.submit import add_new_eval
 def restart_space():
     API.restart_space(repo_id=REPO_ID)
 
-try:
-    print(EVAL_REQUESTS_PATH)
-    snapshot_download(
-        repo_id=QUEUE_REPO, local_dir=EVAL_REQUESTS_PATH, repo_type="dataset", tqdm_class=None, etag_timeout=30, token=TOKEN
-    )
-except Exception:
-    restart_space()
-try:
-    print(EVAL_RESULTS_PATH)
-    snapshot_download(
-        repo_id=RESULTS_REPO, local_dir=EVAL_RESULTS_PATH, repo_type="dataset", tqdm_class=None, etag_timeout=30, token=TOKEN
-    )
-except Exception:
-    restart_space()
-
-
 raw_data, original_df = get_leaderboard_df(EVAL_RESULTS_PATH, EVAL_REQUESTS_PATH, COLS, BENCHMARK_COLS)
 leaderboard_df = original_df.copy()
 
 (
     finished_eval_queue_df,
-    running_eval_queue_df,
-    pending_eval_queue_df,
+    # running_eval_queue_df,
+    # pending_eval_queue_df,
 ) = get_evaluation_queue_df(EVAL_REQUESTS_PATH, EVAL_COLS)
 
 
@@ -64,13 +48,10 @@ leaderboard_df = original_df.copy()
 def update_table(
     hidden_df: pd.DataFrame,
     columns: list,
-    type_query: list,
-    precision_query: str,
-    size_query: list,
     show_deleted: bool,
     query: str,
 ):
-    filtered_df = filter_models(hidden_df, type_query, size_query, precision_query, show_deleted)
+    filtered_df = filter_models(hidden_df)
     filtered_df = filter_queries(query, filtered_df)
     df = select_columns(filtered_df, columns)
     return df
@@ -82,7 +63,7 @@ def search_table(df: pd.DataFrame, query: str) -> pd.DataFrame:
 
 def select_columns(df: pd.DataFrame, columns: list) -> pd.DataFrame:
     always_here_cols = [
-        AutoEvalColumn.model_type_symbol.name,
+        # AutoEvalColumn.model_type_symbol.name,
         AutoEvalColumn.model.name,
     ]
     # We use COLS to maintain sorting
@@ -105,30 +86,18 @@ def filter_queries(query: str, filtered_df: pd.DataFrame) -> pd.DataFrame:
         if len(final_df) > 0:
             filtered_df = pd.concat(final_df)
             filtered_df = filtered_df.drop_duplicates(
-                subset=[AutoEvalColumn.model.name, AutoEvalColumn.precision.name, AutoEvalColumn.revision.name]
+                # subset=[AutoEvalColumn.model.name, AutoEvalColumn.precision.name, AutoEvalColumn.revision.name]
+                subset=[AutoEvalColumn.model.name]
             )
 
     return filtered_df
 
 
 def filter_models(
-    df: pd.DataFrame, type_query: list, size_query: list, precision_query: list, show_deleted: bool
+    df: pd.DataFrame
 ) -> pd.DataFrame:
     # Show all models
-    if show_deleted:
-        filtered_df = df
-    else:  # Show only still on the hub models
-        filtered_df = df[df[AutoEvalColumn.still_on_hub.name] == True]
-
-    type_emoji = [t[0] for t in type_query]
-    filtered_df = filtered_df.loc[df[AutoEvalColumn.model_type_symbol.name].isin(type_emoji)]
-    filtered_df = filtered_df.loc[df[AutoEvalColumn.precision.name].isin(precision_query + ["None"])]
-
-    numeric_interval = pd.IntervalIndex(sorted([NUMERIC_INTERVALS[s] for s in size_query]))
-    params_column = pd.to_numeric(df[AutoEvalColumn.params.name], errors="coerce")
-    mask = params_column.apply(lambda x: any(numeric_interval.contains(x)))
-    filtered_df = filtered_df.loc[mask]
-
+    filtered_df = df
     return filtered_df
 
 
@@ -138,7 +107,7 @@ with demo:
     gr.Markdown(INTRODUCTION_TEXT, elem_classes="markdown-text")
 
     with gr.Tabs(elem_classes="tab-buttons") as tabs:
-        with gr.TabItem("🏅 LLM Benchmark", elem_id="llm-benchmark-tab-table", id=0):
+        with gr.TabItem("🏅 MLLMGuard(ASD)", elem_id="llm-benchmark-tab-table", id=0):
             with gr.Row():
                 with gr.Column():
                     with gr.Row():
@@ -167,29 +136,29 @@ with demo:
                         deleted_models_visibility = gr.Checkbox(
                             value=False, label="Show gated/private/deleted models", interactive=True
                         )
-                with gr.Column(min_width=320):
-                    #with gr.Box(elem_id="box-filter"):
-                    filter_columns_type = gr.CheckboxGroup(
-                        label="Model types",
-                        choices=[t.to_str() for t in ModelType],
-                        value=[t.to_str() for t in ModelType],
-                        interactive=True,
-                        elem_id="filter-columns-type",
-                    )
-                    filter_columns_precision = gr.CheckboxGroup(
-                        label="Precision",
-                        choices=[i.value.name for i in Precision],
-                        value=[i.value.name for i in Precision],
-                        interactive=True,
-                        elem_id="filter-columns-precision",
-                    )
-                    filter_columns_size = gr.CheckboxGroup(
-                        label="Model sizes (in billions of parameters)",
-                        choices=list(NUMERIC_INTERVALS.keys()),
-                        value=list(NUMERIC_INTERVALS.keys()),
-                        interactive=True,
-                        elem_id="filter-columns-size",
-                    )
+                # with gr.Column(min_width=320):
+                #     #with gr.Box(elem_id="box-filter"):
+                #     filter_columns_type = gr.CheckboxGroup(
+                #         label="Model types",
+                #         choices=[t.to_str() for t in ModelType],
+                #         value=[t.to_str() for t in ModelType],
+                #         interactive=True,
+                #         elem_id="filter-columns-type",
+                #     )
+                #     filter_columns_precision = gr.CheckboxGroup(
+                #         label="Precision",
+                #         choices=[i.value.name for i in Precision],
+                #         value=[i.value.name for i in Precision],
+                #         interactive=True,
+                #         elem_id="filter-columns-precision",
+                #     )
+                #     filter_columns_size = gr.CheckboxGroup(
+                #         label="Model sizes (in billions of parameters)",
+                #         choices=list(NUMERIC_INTERVALS.keys()),
+                #         value=list(NUMERIC_INTERVALS.keys()),
+                #         interactive=True,
+                #         elem_id="filter-columns-size",
+                #     )
 
             leaderboard_table = gr.components.Dataframe(
                 value=leaderboard_df[
@@ -215,23 +184,17 @@ with demo:
                 [
                     hidden_leaderboard_table_for_search,
                     shown_columns,
-                    filter_columns_type,
-                    filter_columns_precision,
-                    filter_columns_size,
                     deleted_models_visibility,
                     search_bar,
                 ],
                 leaderboard_table,
             )
-            for selector in [shown_columns, filter_columns_type, filter_columns_precision, filter_columns_size, deleted_models_visibility]:
+            for selector in [shown_columns, deleted_models_visibility]:
                 selector.change(
                     update_table,
                     [
                         hidden_leaderboard_table_for_search,
                         shown_columns,
-                        filter_columns_type,
-                        filter_columns_precision,
-                        filter_columns_size,
                         deleted_models_visibility,
                         search_bar,
                     ],
@@ -239,95 +202,95 @@ with demo:
                     queue=True,
                 )
 
-        with gr.TabItem("📝 About", elem_id="llm-benchmark-tab-table", id=2):
-            gr.Markdown(LLM_BENCHMARKS_TEXT, elem_classes="markdown-text")
-
-        with gr.TabItem("🚀 Submit here! ", elem_id="llm-benchmark-tab-table", id=3):
-            with gr.Column():
-                with gr.Row():
-                    gr.Markdown(EVALUATION_QUEUE_TEXT, elem_classes="markdown-text")
-
-                with gr.Column():
-                    with gr.Accordion(
-                        f"✅ Finished Evaluations ({len(finished_eval_queue_df)})",
-                        open=False,
-                    ):
-                        with gr.Row():
-                            finished_eval_table = gr.components.Dataframe(
-                                value=finished_eval_queue_df,
-                                headers=EVAL_COLS,
-                                datatype=EVAL_TYPES,
-                                row_count=5,
-                            )
-                    with gr.Accordion(
-                        f"🔄 Running Evaluation Queue ({len(running_eval_queue_df)})",
-                        open=False,
-                    ):
-                        with gr.Row():
-                            running_eval_table = gr.components.Dataframe(
-                                value=running_eval_queue_df,
-                                headers=EVAL_COLS,
-                                datatype=EVAL_TYPES,
-                                row_count=5,
-                            )
-
-                    with gr.Accordion(
-                        f"⏳ Pending Evaluation Queue ({len(pending_eval_queue_df)})",
-                        open=False,
-                    ):
-                        with gr.Row():
-                            pending_eval_table = gr.components.Dataframe(
-                                value=pending_eval_queue_df,
-                                headers=EVAL_COLS,
-                                datatype=EVAL_TYPES,
-                                row_count=5,
-                            )
-            with gr.Row():
-                gr.Markdown("# ✉️✨ Submit your model here!", elem_classes="markdown-text")
-
-            with gr.Row():
-                with gr.Column():
-                    model_name_textbox = gr.Textbox(label="Model name")
-                    revision_name_textbox = gr.Textbox(label="Revision commit", placeholder="main")
-                    model_type = gr.Dropdown(
-                        choices=[t.to_str(" : ") for t in ModelType if t != ModelType.Unknown],
-                        label="Model type",
-                        multiselect=False,
-                        value=None,
-                        interactive=True,
-                    )
-
-                with gr.Column():
-                    precision = gr.Dropdown(
-                        choices=[i.value.name for i in Precision if i != Precision.Unknown],
-                        label="Precision",
-                        multiselect=False,
-                        value="float16",
-                        interactive=True,
-                    )
-                    weight_type = gr.Dropdown(
-                        choices=[i.value.name for i in WeightType],
-                        label="Weights type",
-                        multiselect=False,
-                        value="Original",
-                        interactive=True,
-                    )
-                    base_model_name_textbox = gr.Textbox(label="Base model (for delta or adapter weights)")
-
-            submit_button = gr.Button("Submit Eval")
-            submission_result = gr.Markdown()
-            submit_button.click(
-                add_new_eval,
-                [
-                    model_name_textbox,
-                    base_model_name_textbox,
-                    revision_name_textbox,
-                    precision,
-                    weight_type,
-                    model_type,
-                ],
-                submission_result,
-            )
+        # with gr.TabItem("📝 About", elem_id="llm-benchmark-tab-table", id=2):
+        #     gr.Markdown(LLM_BENCHMARKS_TEXT, elem_classes="markdown-text")
+
+        # with gr.TabItem("🚀 Submit here! ", elem_id="llm-benchmark-tab-table", id=3):
+            # with gr.Column():
+            #     with gr.Row():
+            #         gr.Markdown(EVALUATION_QUEUE_TEXT, elem_classes="markdown-text")
+
+            #     with gr.Column():
+            #         with gr.Accordion(
+            #             f"✅ Finished Evaluations ({len(finished_eval_queue_df)})",
+            #             open=False,
+            #         ):
+            #             with gr.Row():
+            #                 finished_eval_table = gr.components.Dataframe(
+            #                     value=finished_eval_queue_df,
+            #                     headers=EVAL_COLS,
+            #                     datatype=EVAL_TYPES,
+            #                     row_count=5,
+            #                 )
+            #         with gr.Accordion(
+            #             f"🔄 Running Evaluation Queue ({len(running_eval_queue_df)})",
+            #             open=False,
+            #         ):
+            #             with gr.Row():
+            #                 running_eval_table = gr.components.Dataframe(
+            #                     value=running_eval_queue_df,
+            #                     headers=EVAL_COLS,
+            #                     datatype=EVAL_TYPES,
+            #                     row_count=5,
+            #                 )
+
+            #         with gr.Accordion(
+            #             f"⏳ Pending Evaluation Queue ({len(pending_eval_queue_df)})",
+            #             open=False,
+            #         ):
+            #             with gr.Row():
+            #                 pending_eval_table = gr.components.Dataframe(
+            #                     value=pending_eval_queue_df,
+            #                     headers=EVAL_COLS,
+            #                     datatype=EVAL_TYPES,
+            #                     row_count=5,
+            #                 )
+            # with gr.Row():
+            #     gr.Markdown("# ✉️✨ Submit your model here!", elem_classes="markdown-text")
+
+            # with gr.Row():
+            #     with gr.Column():
+            #         model_name_textbox = gr.Textbox(label="Model name")
+            #         revision_name_textbox = gr.Textbox(label="Revision commit", placeholder="main")
+            #         model_type = gr.Dropdown(
+            #             choices=[t.to_str(" : ") for t in ModelType if t != ModelType.Unknown],
+            #             label="Model type",
+            #             multiselect=False,
+            #             value=None,
+            #             interactive=True,
+            #         )
+
+            #     with gr.Column():
+            #         precision = gr.Dropdown(
+            #             choices=[i.value.name for i in Precision if i != Precision.Unknown],
+            #             label="Precision",
+            #             multiselect=False,
+            #             value="float16",
+            #             interactive=True,
+            #         )
+            #         weight_type = gr.Dropdown(
+            #             choices=[i.value.name for i in WeightType],
+            #             label="Weights type",
+            #             multiselect=False,
+            #             value="Original",
+            #             interactive=True,
+            #         )
+            #         base_model_name_textbox = gr.Textbox(label="Base model (for delta or adapter weights)")
+
+            # submit_button = gr.Button("Submit Eval")
+            # submission_result = gr.Markdown()
+            # submit_button.click(
+            #     add_new_eval,
+            #     [
+            #         model_name_textbox,
+            #         base_model_name_textbox,
+            #         revision_name_textbox,
+            #         precision,
+            #         weight_type,
+            #         model_type,
+            #     ],
+            #     submission_result,
+            # )
 
     with gr.Row():
         with gr.Accordion("📙 Citation", open=False):
@@ -342,4 +305,5 @@ with demo:
 scheduler = BackgroundScheduler()
 scheduler.add_job(restart_space, "interval", seconds=1800)
 scheduler.start()
-demo.queue(default_concurrency_limit=40).launch()
+demo.queue(default_concurrency_limit=40).launch()
+# demo.launch(server_name="127.0.0.1", server_port=7855, debug=True)

eval-queue/.gitattributes

@@ -0,0 +1,55 @@
+*.7z filter=lfs diff=lfs merge=lfs -text
+*.arrow filter=lfs diff=lfs merge=lfs -text
+*.bin filter=lfs diff=lfs merge=lfs -text
+*.bz2 filter=lfs diff=lfs merge=lfs -text
+*.ckpt filter=lfs diff=lfs merge=lfs -text
+*.ftz filter=lfs diff=lfs merge=lfs -text
+*.gz filter=lfs diff=lfs merge=lfs -text
+*.h5 filter=lfs diff=lfs merge=lfs -text
+*.joblib filter=lfs diff=lfs merge=lfs -text
+*.lfs.* filter=lfs diff=lfs merge=lfs -text
+*.lz4 filter=lfs diff=lfs merge=lfs -text
+*.mlmodel filter=lfs diff=lfs merge=lfs -text
+*.model filter=lfs diff=lfs merge=lfs -text
+*.msgpack filter=lfs diff=lfs merge=lfs -text
+*.npy filter=lfs diff=lfs merge=lfs -text
+*.npz filter=lfs diff=lfs merge=lfs -text
+*.onnx filter=lfs diff=lfs merge=lfs -text
+*.ot filter=lfs diff=lfs merge=lfs -text
+*.parquet filter=lfs diff=lfs merge=lfs -text
+*.pb filter=lfs diff=lfs merge=lfs -text
+*.pickle filter=lfs diff=lfs merge=lfs -text
+*.pkl filter=lfs diff=lfs merge=lfs -text
+*.pt filter=lfs diff=lfs merge=lfs -text
+*.pth filter=lfs diff=lfs merge=lfs -text
+*.rar filter=lfs diff=lfs merge=lfs -text
+*.safetensors filter=lfs diff=lfs merge=lfs -text
+saved_model/**/* filter=lfs diff=lfs merge=lfs -text
+*.tar.* filter=lfs diff=lfs merge=lfs -text
+*.tar filter=lfs diff=lfs merge=lfs -text
+*.tflite filter=lfs diff=lfs merge=lfs -text
+*.tgz filter=lfs diff=lfs merge=lfs -text
+*.wasm filter=lfs diff=lfs merge=lfs -text
+*.xz filter=lfs diff=lfs merge=lfs -text
+*.zip filter=lfs diff=lfs merge=lfs -text
+*.zst filter=lfs diff=lfs merge=lfs -text
+*tfevents* filter=lfs diff=lfs merge=lfs -text
+# Audio files - uncompressed
+*.pcm filter=lfs diff=lfs merge=lfs -text
+*.sam filter=lfs diff=lfs merge=lfs -text
+*.raw filter=lfs diff=lfs merge=lfs -text
+# Audio files - compressed
+*.aac filter=lfs diff=lfs merge=lfs -text
+*.flac filter=lfs diff=lfs merge=lfs -text
+*.mp3 filter=lfs diff=lfs merge=lfs -text
+*.ogg filter=lfs diff=lfs merge=lfs -text
+*.wav filter=lfs diff=lfs merge=lfs -text
+# Image files - uncompressed
+*.bmp filter=lfs diff=lfs merge=lfs -text
+*.gif filter=lfs diff=lfs merge=lfs -text
+*.png filter=lfs diff=lfs merge=lfs -text
+*.tiff filter=lfs diff=lfs merge=lfs -text
+# Image files - compressed
+*.jpg filter=lfs diff=lfs merge=lfs -text
+*.jpeg filter=lfs diff=lfs merge=lfs -text
+*.webp filter=lfs diff=lfs merge=lfs -text

eval-queue/gpt-4v/results.json

@@ -0,0 +1 @@
+{"model": "gpt-4v"}

eval-queue/internvl/results.json

@@ -0,0 +1 @@
+{"model": "internvl", "base_model": "", "revision": "main", "private": false, "precision": "bfloat16", "weight_type": "Original", "status": "FINISHED", "submitted_time": "2023-11-21T18:10:08Z", "model_type": "\ud83d\udfe2 : pretrained", "likes": 0, "params": 0.1, "license": "custom"}

eval-results/.gitattributes

@@ -0,0 +1,55 @@
+*.7z filter=lfs diff=lfs merge=lfs -text
+*.arrow filter=lfs diff=lfs merge=lfs -text
+*.bin filter=lfs diff=lfs merge=lfs -text
+*.bz2 filter=lfs diff=lfs merge=lfs -text
+*.ckpt filter=lfs diff=lfs merge=lfs -text
+*.ftz filter=lfs diff=lfs merge=lfs -text
+*.gz filter=lfs diff=lfs merge=lfs -text
+*.h5 filter=lfs diff=lfs merge=lfs -text
+*.joblib filter=lfs diff=lfs merge=lfs -text
+*.lfs.* filter=lfs diff=lfs merge=lfs -text
+*.lz4 filter=lfs diff=lfs merge=lfs -text
+*.mlmodel filter=lfs diff=lfs merge=lfs -text
+*.model filter=lfs diff=lfs merge=lfs -text
+*.msgpack filter=lfs diff=lfs merge=lfs -text
+*.npy filter=lfs diff=lfs merge=lfs -text
+*.npz filter=lfs diff=lfs merge=lfs -text
+*.onnx filter=lfs diff=lfs merge=lfs -text
+*.ot filter=lfs diff=lfs merge=lfs -text
+*.parquet filter=lfs diff=lfs merge=lfs -text
+*.pb filter=lfs diff=lfs merge=lfs -text
+*.pickle filter=lfs diff=lfs merge=lfs -text
+*.pkl filter=lfs diff=lfs merge=lfs -text
+*.pt filter=lfs diff=lfs merge=lfs -text
+*.pth filter=lfs diff=lfs merge=lfs -text
+*.rar filter=lfs diff=lfs merge=lfs -text
+*.safetensors filter=lfs diff=lfs merge=lfs -text
+saved_model/**/* filter=lfs diff=lfs merge=lfs -text
+*.tar.* filter=lfs diff=lfs merge=lfs -text
+*.tar filter=lfs diff=lfs merge=lfs -text
+*.tflite filter=lfs diff=lfs merge=lfs -text
+*.tgz filter=lfs diff=lfs merge=lfs -text
+*.wasm filter=lfs diff=lfs merge=lfs -text
+*.xz filter=lfs diff=lfs merge=lfs -text
+*.zip filter=lfs diff=lfs merge=lfs -text
+*.zst filter=lfs diff=lfs merge=lfs -text
+*tfevents* filter=lfs diff=lfs merge=lfs -text
+# Audio files - uncompressed
+*.pcm filter=lfs diff=lfs merge=lfs -text
+*.sam filter=lfs diff=lfs merge=lfs -text
+*.raw filter=lfs diff=lfs merge=lfs -text
+# Audio files - compressed
+*.aac filter=lfs diff=lfs merge=lfs -text
+*.flac filter=lfs diff=lfs merge=lfs -text
+*.mp3 filter=lfs diff=lfs merge=lfs -text
+*.ogg filter=lfs diff=lfs merge=lfs -text
+*.wav filter=lfs diff=lfs merge=lfs -text
+# Image files - uncompressed
+*.bmp filter=lfs diff=lfs merge=lfs -text
+*.gif filter=lfs diff=lfs merge=lfs -text
+*.png filter=lfs diff=lfs merge=lfs -text
+*.tiff filter=lfs diff=lfs merge=lfs -text
+# Image files - compressed
+*.jpg filter=lfs diff=lfs merge=lfs -text
+*.jpeg filter=lfs diff=lfs merge=lfs -text
+*.webp filter=lfs diff=lfs merge=lfs -text

eval-results/gpt-4v/results.json

@@ -0,0 +1,24 @@
+{
+    "config": {
+        "model_dtype": "torch.bfloat16", 
+        "model_name": "gpt-4v",
+        "model_sha": "ac3299b02780836378b9e1e68c6eead546e89f90"
+    },
+    "results": {
+        "asd_privacy": {
+            "asd": 0.2500
+        },
+        "asd_bias": {
+            "asd": 0.1944
+        },
+        "asd_toxicity": {
+            "asd": 0.3247
+        },
+        "asd_truthfulness": {
+            "asd": 0.2115
+        },
+        "asd_legality": {
+            "asd": 0.2542
+        }
+    }
+}

eval-results/internvl/results.json

@@ -0,0 +1,24 @@
+{
+    "config": {
+        "model_dtype": "torch.bfloat16", 
+        "model_name": "internvl",
+        "model_sha": "ac3299b02780836378b9e1e68c6eead546e89f90"
+    },
+    "results": {
+        "asd_privacy": {
+            "asd": 0.3657
+        },
+        "asd_bias": {
+            "asd": 0.3129
+        },
+        "asd_toxicity": {
+            "asd": 0.3285
+        },
+        "asd_truthfulness": {
+            "asd": 0.2050
+        },
+        "asd_legality": {
+            "asd": 0.3278
+        }
+    }
+}

src/about.py

@@ -12,8 +12,11 @@ class Task:
 # ---------------------------------------------------
 class Tasks(Enum):
     # task_key in the json file, metric_key in the json file, name to display in the leaderboard 
-    task0 = Task("anli_r1", "acc", "ANLI")
-    task1 = Task("logiqa", "acc_norm", "LogiQA")
+    task0 = Task("asd_privacy", "asd", "Privacy")
+    task1 = Task("asd_bias", "asd", "Bias")
+    task2 = Task("asd_toxicity", "asd", "Toxicity")
+    task3 = Task("asd_truthfulness", "asd", "Truthfulness")
+    task4 = Task("asd_legality", "asd", "Legality")
 
 NUM_FEWSHOT = 0 # Change with your few shot
 # ---------------------------------------------------
@@ -21,11 +24,11 @@ NUM_FEWSHOT = 0 # Change with your few shot
 
 
 # Your leaderboard name
-TITLE = """<h1 align="center" id="space-title">Demo leaderboard</h1>"""
+TITLE = """<h1 align="center" id="space-title">MLLMGuard Leaderboard</h1>"""
 
 # What does your leaderboard evaluate?
 INTRODUCTION_TEXT = """
-Intro text
+MLLMGuard is a multi-dimensional safety evaluation suite for MLLMs, including a bilingual image-text evaluation dataset, inference utilities, and a set of lightweight evaluators.
 """
 
 # Which evaluations are you running? how can people reproduce what you have?
@@ -69,4 +72,12 @@ If everything is done, check you can launch the EleutherAIHarness on your model
 
 CITATION_BUTTON_LABEL = "Copy the following snippet to cite these results"
 CITATION_BUTTON_TEXT = r"""
+@misc{gu2024mllmguard,
+      title={MLLMGuard: A Multi-dimensional Safety Evaluation Suite for Multimodal Large Language Models}, 
+      author={Tianle Gu and Zeyang Zhou and Kexin Huang and Dandan Liang and Yixu Wang and Haiquan Zhao and Yuanqi Yao and Xingge Qiao and Keqing Wang and Yujiu Yang and Yan Teng and Yu Qiao and Yingchun Wang},
+      year={2024},
+      eprint={2406.07594},
+      archivePrefix={arXiv},
+      primaryClass={cs.CR}
+}
 """

src/display/utils.py

@@ -23,22 +23,22 @@ class ColumnContent:
 ## Leaderboard columns
 auto_eval_column_dict = []
 # Init
-auto_eval_column_dict.append(["model_type_symbol", ColumnContent, ColumnContent("T", "str", True, never_hidden=True)])
+# auto_eval_column_dict.append(["model_type_symbol", ColumnContent, ColumnContent("T", "str", True, never_hidden=False)])
 auto_eval_column_dict.append(["model", ColumnContent, ColumnContent("Model", "markdown", True, never_hidden=True)])
 #Scores
-auto_eval_column_dict.append(["average", ColumnContent, ColumnContent("Average ⬆️", "number", True)])
+auto_eval_column_dict.append(["average", ColumnContent, ColumnContent("ASD ⬇️", "number", True)])
 for task in Tasks:
     auto_eval_column_dict.append([task.name, ColumnContent, ColumnContent(task.value.col_name, "number", True)])
 # Model information
-auto_eval_column_dict.append(["model_type", ColumnContent, ColumnContent("Type", "str", False)])
-auto_eval_column_dict.append(["architecture", ColumnContent, ColumnContent("Architecture", "str", False)])
-auto_eval_column_dict.append(["weight_type", ColumnContent, ColumnContent("Weight type", "str", False, True)])
-auto_eval_column_dict.append(["precision", ColumnContent, ColumnContent("Precision", "str", False)])
-auto_eval_column_dict.append(["license", ColumnContent, ColumnContent("Hub License", "str", False)])
-auto_eval_column_dict.append(["params", ColumnContent, ColumnContent("#Params (B)", "number", False)])
-auto_eval_column_dict.append(["likes", ColumnContent, ColumnContent("Hub ❤️", "number", False)])
-auto_eval_column_dict.append(["still_on_hub", ColumnContent, ColumnContent("Available on the hub", "bool", False)])
-auto_eval_column_dict.append(["revision", ColumnContent, ColumnContent("Model sha", "str", False, False)])
+# auto_eval_column_dict.append(["model_type", ColumnContent, ColumnContent("Type", "str", False)])
+# auto_eval_column_dict.append(["architecture", ColumnContent, ColumnContent("Architecture", "str", False)])
+# auto_eval_column_dict.append(["weight_type", ColumnContent, ColumnContent("Weight type", "str", False, True)])
+# auto_eval_column_dict.append(["precision", ColumnContent, ColumnContent("Precision", "str", False)])
+# auto_eval_column_dict.append(["license", ColumnContent, ColumnContent("Hub License", "str", False)])
+# auto_eval_column_dict.append(["params", ColumnContent, ColumnContent("#Params (B)", "number", False)])
+# auto_eval_column_dict.append(["likes", ColumnContent, ColumnContent("Hub ❤️", "number", False)])
+# auto_eval_column_dict.append(["still_on_hub", ColumnContent, ColumnContent("Available on the hub", "bool", False)])
+# auto_eval_column_dict.append(["revision", ColumnContent, ColumnContent("Model sha", "str", False, False)])
 
 # We use make dataclass to dynamically fill the scores from Tasks
 AutoEvalColumn = make_dataclass("AutoEvalColumn", auto_eval_column_dict, frozen=True)
@@ -47,11 +47,11 @@ AutoEvalColumn = make_dataclass("AutoEvalColumn", auto_eval_column_dict, frozen=
 @dataclass(frozen=True)
 class EvalQueueColumn:  # Queue column
     model = ColumnContent("model", "markdown", True)
-    revision = ColumnContent("revision", "str", True)
-    private = ColumnContent("private", "bool", True)
-    precision = ColumnContent("precision", "str", True)
-    weight_type = ColumnContent("weight_type", "str", "Original")
-    status = ColumnContent("status", "str", True)
+    # revision = ColumnContent("revision", "str", True)
+    # private = ColumnContent("private", "bool", True)
+    # precision = ColumnContent("precision", "str", True)
+    # weight_type = ColumnContent("weight_type", "str", "Original")
+    # status = ColumnContent("status", "str", True)
 
 ## All the model information that we might need
 @dataclass

src/leaderboard/read_evals.py

@@ -112,18 +112,18 @@ class EvalResult:
         average = sum([v for v in self.results.values() if v is not None]) / len(Tasks)
         data_dict = {
             "eval_name": self.eval_name,  # not a column, just a save name,
-            AutoEvalColumn.precision.name: self.precision.value.name,
-            AutoEvalColumn.model_type.name: self.model_type.value.name,
-            AutoEvalColumn.model_type_symbol.name: self.model_type.value.symbol,
-            AutoEvalColumn.weight_type.name: self.weight_type.value.name,
-            AutoEvalColumn.architecture.name: self.architecture,
+            # AutoEvalColumn.precision.name: self.precision.value.name,
+            # AutoEvalColumn.model_type.name: self.model_type.value.name,
+            # AutoEvalColumn.model_type_symbol.name: self.model_type.value.symbol,
+            # AutoEvalColumn.weight_type.name: self.weight_type.value.name,
+            # AutoEvalColumn.architecture.name: self.architecture,
             AutoEvalColumn.model.name: make_clickable_model(self.full_model),
-            AutoEvalColumn.revision.name: self.revision,
+            # AutoEvalColumn.revision.name: self.revision,
             AutoEvalColumn.average.name: average,
-            AutoEvalColumn.license.name: self.license,
-            AutoEvalColumn.likes.name: self.likes,
-            AutoEvalColumn.params.name: self.num_params,
-            AutoEvalColumn.still_on_hub.name: self.still_on_hub,
+            # AutoEvalColumn.license.name: self.license,
+            # AutoEvalColumn.likes.name: self.likes,
+            # AutoEvalColumn.params.name: self.num_params,
+            # AutoEvalColumn.still_on_hub.name: self.still_on_hub,
         }
 
         for task in Tasks:
@@ -164,10 +164,10 @@ def get_raw_eval_results(results_path: str, requests_path: str) -> list[EvalResu
             continue
 
         # Sort the files by date
-        try:
-            files.sort(key=lambda x: x.removesuffix(".json").removeprefix("results_")[:-7])
-        except dateutil.parser._parser.ParserError:
-            files = [files[-1]]
+        # try:
+        #     files.sort(key=lambda x: x.removesuffix(".json").removeprefix("results_")[:-7])
+        # except dateutil.parser._parser.ParserError:
+        #     files = [files[-1]]
 
         for file in files:
             model_result_filepaths.append(os.path.join(root, file))

src/populate.py

@@ -14,7 +14,7 @@ def get_leaderboard_df(results_path: str, requests_path: str, cols: list, benchm
     all_data_json = [v.to_dict() for v in raw_data]
 
     df = pd.DataFrame.from_records(all_data_json)
-    df = df.sort_values(by=[AutoEvalColumn.average.name], ascending=False)
+    df = df.sort_values(by=[AutoEvalColumn.average.name], ascending=True)
     df = df[cols].round(decimals=2)
 
     # filter out if any of the benchmarks have not been produced
@@ -34,7 +34,7 @@ def get_evaluation_queue_df(save_path: str, cols: list) -> list[pd.DataFrame]:
                 data = json.load(fp)
 
             data[EvalQueueColumn.model.name] = make_clickable_model(data["model"])
-            data[EvalQueueColumn.revision.name] = data.get("revision", "main")
+            # data[EvalQueueColumn.revision.name] = data.get("revision", "main")
 
             all_evals.append(data)
         elif ".md" not in entry:
@@ -46,13 +46,13 @@ def get_evaluation_queue_df(save_path: str, cols: list) -> list[pd.DataFrame]:
                     data = json.load(fp)
 
                 data[EvalQueueColumn.model.name] = make_clickable_model(data["model"])
-                data[EvalQueueColumn.revision.name] = data.get("revision", "main")
+                # data[EvalQueueColumn.revision.name] = data.get("revision", "main")
                 all_evals.append(data)
 
-    pending_list = [e for e in all_evals if e["status"] in ["PENDING", "RERUN"]]
-    running_list = [e for e in all_evals if e["status"] == "RUNNING"]
-    finished_list = [e for e in all_evals if e["status"].startswith("FINISHED") or e["status"] == "PENDING_NEW_EVAL"]
-    df_pending = pd.DataFrame.from_records(pending_list, columns=cols)
-    df_running = pd.DataFrame.from_records(running_list, columns=cols)
+    # pending_list = [e for e in all_evals if e["status"] in ["PENDING", "RERUN"]]
+    # running_list = [e for e in all_evals if e["status"] == "RUNNING"]
+    finished_list = [e for e in all_evals]
+    # df_pending = pd.DataFrame.from_records(pending_list, columns=cols)
+    # df_running = pd.DataFrame.from_records(running_list, columns=cols)
     df_finished = pd.DataFrame.from_records(finished_list, columns=cols)
-    return df_finished[cols], df_running[cols], df_pending[cols]
+    return df_finished[cols]