Adding flagging systemi, removing changelog

app.py

@@ -82,7 +82,7 @@ def get_leaderboard_df():
         print("Pulling evaluation results for the leaderboard.")
         eval_results_private.git_pull()
 
-    all_data = get_eval_results_dicts(IS_PUBLIC)
+    all_data = get_eval_results_dicts()
 
     if not IS_PUBLIC:
         all_data.append(gpt4_values)
@@ -341,7 +341,7 @@ with demo:
                         elem_id="filter-columns"
                     )
             leaderboard_table = gr.components.Dataframe(
-                value=leaderboard_df[[AutoEvalColumn.model_type_symbol.name, AutoEvalColumn.model.name] + shown_columns.value+ [AutoEvalColumn.dummy.name]],
+                value=leaderboard_df[[AutoEvalColumn.model_type_symbol.name, AutoEvalColumn.model.name] + shown_columns.value + [AutoEvalColumn.dummy.name]],
                 headers=[AutoEvalColumn.model_type_symbol.name, AutoEvalColumn.model.name] + shown_columns.value + [AutoEvalColumn.dummy.name],
                 datatype=TYPES,
                 max_rows=None,

src/assets/css_html_js.py

@@ -1,11 +1,4 @@
 custom_css = """
-#changelog-text {
-    font-size: 16px !important;
-}
-
-#changelog-text h2 {
-    font-size: 18px !important;
-}
 
 .markdown-text {
     font-size: 16px !important;

src/assets/text_content.py

@@ -1,61 +1,5 @@
 from ..auto_leaderboard.model_metadata_type import ModelType
 
-CHANGELOG_TEXT = f"""
-## [2023-06-19]
-- Added model type column
-- Hid revision and 8bit columns since all models are the same atm
-
-## [2023-06-16]
-- Refactored code base
-- Added new columns: number of parameters, hub likes, license
-
-## [2023-06-13] 
-- Adjust description for TruthfulQA
-
-## [2023-06-12] 
-- Add Human & GPT-4 Evaluations
-
-## [2023-06-05] 
-- Increase concurrent thread count to 40
-- Search models on ENTER
-
-## [2023-06-02] 
-- Add a typeahead search bar
-- Use webhooks to automatically spawn a new Space when someone opens a PR
-- Start recording `submitted_time` for eval requests
-- Limit AutoEvalColumn max-width
-
-## [2023-05-30] 
-- Add a citation button
-- Simplify Gradio layout
-
-## [2023-05-29] 
-- Auto-restart every hour for the latest results
-- Sync with the internal version (minor style changes)
-
-## [2023-05-24] 
-- Add a baseline that has 25.0 for all values
-- Add CHANGELOG
-
-## [2023-05-23] 
-- Fix a CSS issue that made the leaderboard hard to read in dark mode
-
-## [2023-05-22] 
-- Display a success/error message after submitting evaluation requests
-- Reject duplicate submission
-- Do not display results that have incomplete results 
-- Display different queues for jobs that are RUNNING, PENDING, FINISHED status 
-
-## [2023-05-15] 
-- Fix a typo: from "TruthQA" to "QA"
-
-## [2023-05-10] 
-- Fix a bug that prevented auto-refresh
-
-## [2023-05-10] 
-- Release the leaderboard to public
-"""
-
 TITLE = """<h1 align="center" id="space-title">🤗 Open LLM Leaderboard</h1>"""
 
 INTRODUCTION_TEXT = f"""
@@ -81,6 +25,9 @@ With the plethora of large language models (LLMs) and chatbots being released we
 {ModelType.RL.to_str(" : ")} model
 If there is no icon, we have not uploaded the information on the model yet, feel free to open an issue with the model information!
 
+🏴‍☠️ indicates that this model has been flagged by the community, and should probably be ignored! Clicking the icon will redirect you to the discussion about the model.
+(For ex, the model was trained on the evaluation data, and is therefore cheating on the leaderboard.)
+
 ## How it works
 
 📈 We evaluate models on 4 key benchmarks using the <a href="https://github.com/EleutherAI/lm-evaluation-harness" target="_blank">  Eleuther AI Language Model Evaluation Harness </a>, a unified framework to test generative language models on a large number of different evaluation tasks. 

src/auto_leaderboard/get_model_metadata.py

@@ -1,10 +1,14 @@
 import re
 import os
+import glob
+import json
+import os
 from typing import List
 from tqdm import tqdm
 
-from src.utils_display import AutoEvalColumn
-from src.auto_leaderboard.model_metadata_type import get_model_type
+from src.utils_display import AutoEvalColumn, model_hyperlink
+from src.auto_leaderboard.model_metadata_type import ModelType, model_type_from_str, MODEL_TYPE_METADATA
+from src.auto_leaderboard.model_metadata_flags import FLAGGED_MODELS
 
 from huggingface_hub import HfApi
 import huggingface_hub
@@ -52,6 +56,60 @@ def get_model_size(model_name, model_info):
             return None
 
 
+def get_model_type(leaderboard_data: List[dict]):
+    for model_data in leaderboard_data:
+        request_files = os.path.join("eval-queue", model_data["model_name_for_query"] + "_eval_request_*" + ".json")
+        request_files = glob.glob(request_files)
+
+        # Select correct request file (precision)
+        request_file = ""
+        if len(request_files) == 1:
+            request_file = request_files[0]
+        elif len(request_files) > 1:
+            request_files = sorted(request_files, reverse=True)
+            for tmp_request_file in request_files:
+                with open(tmp_request_file, "r") as f:
+                    req_content = json.load(f)
+                    if req_content["status"] == "FINISHED" and req_content["precision"] == model_data["Precision"].split(".")[-1]: 
+                        request_file = tmp_request_file
+        
+        if request_file == "":
+            model_data[AutoEvalColumn.model_type.name] = ""
+            model_data[AutoEvalColumn.model_type_symbol.name] = ""
+            continue
+
+        try:
+            with open(request_file, "r") as f:
+                request = json.load(f)
+            is_delta = request["weight_type"] != "Original"
+        except Exception:
+            is_delta = False
+
+        try:
+            with open(request_file, "r") as f:
+                request = json.load(f)
+            model_type = model_type_from_str(request["model_type"])
+            model_data[AutoEvalColumn.model_type.name] = model_type.value.name
+            model_data[AutoEvalColumn.model_type_symbol.name] = model_type.value.symbol #+ ("🔺" if is_delta else "")
+        except KeyError:
+            if model_data["model_name_for_query"] in MODEL_TYPE_METADATA:
+                model_data[AutoEvalColumn.model_type.name] = MODEL_TYPE_METADATA[model_data["model_name_for_query"]].value.name
+                model_data[AutoEvalColumn.model_type_symbol.name] = MODEL_TYPE_METADATA[model_data["model_name_for_query"]].value.symbol #+ ("🔺" if is_delta else "")
+            else:
+                model_data[AutoEvalColumn.model_type.name] = ModelType.Unknown.value.name
+                model_data[AutoEvalColumn.model_type_symbol.name] = ModelType.Unknown.value.symbol
+
+def flag_models(leaderboard_data:List[dict]):
+    flag_symbol = "💀"
+    for model_data in leaderboard_data:
+        if model_data["model_name_for_query"] in FLAGGED_MODELS:
+            issue_num = FLAGGED_MODELS[model_data["model_name_for_query"]].split("/")[-1]
+            issue_link = model_hyperlink(FLAGGED_MODELS[model_data["model_name_for_query"]], f"See discussion #{issue_num}")
+
+            model_data[AutoEvalColumn.model_type_symbol.name] = flag_symbol
+            model_data[AutoEvalColumn.model.name] =  f"{model_data[AutoEvalColumn.model.name]} has been flagged! {issue_link}"
+
 def apply_metadata(leaderboard_data: List[dict]):
     get_model_type(leaderboard_data)
     get_model_infos_from_hub(leaderboard_data)
+    flag_models(leaderboard_data)

src/auto_leaderboard/load_results.py

@@ -102,7 +102,7 @@ def parse_eval_result(json_filepath: str) -> Tuple[str, list[dict]]:
     return result_key, eval_results
 
 
-def get_eval_results(is_public) -> List[EvalResult]:
+def get_eval_results() -> List[EvalResult]:
     json_filepaths = []
 
     for root, dir, files in os.walk("eval-results"):
@@ -135,7 +135,7 @@ def get_eval_results(is_public) -> List[EvalResult]:
     return eval_results
 
 
-def get_eval_results_dicts(is_public=True) -> List[Dict]:
-    eval_results = get_eval_results(is_public)
+def get_eval_results_dicts() -> List[Dict]:
+    eval_results = get_eval_results()
 
     return [e.to_dict() for e in eval_results]

src/auto_leaderboard/model_metadata_flags.py

@@ -0,0 +1,5 @@
+# Model name to forum discussion id
+FLAGGED_MODELS = {
+    "Voicelab/trurl-2-13b": "https://huggingface.co/spaces/HuggingFaceH4/open_llm_leaderboard/discussions/202",
+    "deepnight-research/llama-2-70B-inst": "https://huggingface.co/spaces/HuggingFaceH4/open_llm_leaderboard/discussions/207"
+}

src/auto_leaderboard/model_metadata_type.py

@@ -1,11 +1,7 @@
 from dataclasses import dataclass
 from enum import Enum
-import glob
-import json
-import os
-from typing import Dict, List
+from typing import Dict
 
-from ..utils_display import AutoEvalColumn
 
 @dataclass
 class ModelInfo:
@@ -24,7 +20,7 @@ class ModelType(Enum):
         return f"{self.value.symbol}{separator}{self.value.name}" 
 
 
-TYPE_METADATA: Dict[str, ModelType] = {
+MODEL_TYPE_METADATA: Dict[str, ModelType] = {
     'notstoic/PygmalionCoT-7b': ModelType.IFT,
     'aisquared/dlite-v1-355m': ModelType.IFT,
     'aisquared/dlite-v1-1_5b': ModelType.IFT,
@@ -553,45 +549,3 @@ def model_type_from_str(type):
         return ModelType.IFT
     return ModelType.Unknown
 
-
-def get_model_type(leaderboard_data: List[dict]):
-    for model_data in leaderboard_data:
-        request_files = os.path.join("eval-queue", model_data["model_name_for_query"] + "_eval_request_*" + ".json")
-        request_files = glob.glob(request_files)
-
-        request_file = ""
-        if len(request_files) == 1:
-            request_file = request_files[0]
-        elif len(request_files) > 1:
-            request_files = sorted(request_files, reverse=True)
-            for tmp_request_file in request_files:
-                with open(tmp_request_file, "r") as f:
-                    req_content = json.load(f)
-                    if req_content["status"] == "FINISHED" and req_content["precision"] == model_data["Precision"].split(".")[-1]: 
-                        request_file = tmp_request_file
-        
-        if request_file == "":
-            model_data[AutoEvalColumn.model_type.name] = ""
-            model_data[AutoEvalColumn.model_type_symbol.name] = ""
-            continue
-
-        try:
-            with open(request_file, "r") as f:
-                request = json.load(f)
-            is_delta = request["weight_type"] != "Original"
-        except Exception:
-            is_delta = False
-
-        try:
-            with open(request_file, "r") as f:
-                request = json.load(f)
-            model_type = model_type_from_str(request["model_type"])
-            model_data[AutoEvalColumn.model_type.name] = model_type.value.name
-            model_data[AutoEvalColumn.model_type_symbol.name] = model_type.value.symbol #+ ("🔺" if is_delta else "")
-        except KeyError:
-            if model_data["model_name_for_query"] in TYPE_METADATA:
-                model_data[AutoEvalColumn.model_type.name] = TYPE_METADATA[model_data["model_name_for_query"]].value.name
-                model_data[AutoEvalColumn.model_type_symbol.name] = TYPE_METADATA[model_data["model_name_for_query"]].value.symbol #+ ("🔺" if is_delta else "")
-            else:
-                model_data[AutoEvalColumn.model_type.name] = ModelType.Unknown.value.name
-                model_data[AutoEvalColumn.model_type_symbol.name] = ModelType.Unknown.value.symbol

src/utils_display.py

@@ -89,20 +89,22 @@ def make_clickable_model(model_name):
         link = KOALA_LINK
     elif model_name == "oasst-12b":
         link = OASST_LINK
-    #else:
-    #    link = MODEL_PAGE
+
     details_model_name = model_name.replace('/', '__')
     details_link = f"https://huggingface.co/datasets/open-llm-leaderboard/details_{details_model_name}"
-    print(f"details_link: {details_link}")
-    try:
-        check_path = list(API.list_files_info(repo_id=f"open-llm-leaderboard/details_{details_model_name}",
-                                              paths="README.md",
-                                              repo_type="dataset"))
-        print(f"check_path: {check_path}")
-    except Exception as err:
-        # No details repo for this model
-        print(f"No details repo for this model: {err}")
-        return model_hyperlink(link, model_name)
+
+    if not bool(os.getenv("DEBUG", "False")):
+        # We only add these checks when not debugging, as they are extremely slow
+        print(f"details_link: {details_link}")
+        try:
+            check_path = list(API.list_files_info(repo_id=f"open-llm-leaderboard/details_{details_model_name}",
+                                                paths="README.md",
+                                                repo_type="dataset"))
+            print(f"check_path: {check_path}")
+        except Exception as err:
+            # No details repo for this model
+            print(f"No details repo for this model: {err}")
+            return model_hyperlink(link, model_name)
 
     return model_hyperlink(link, model_name) + '  ' + model_hyperlink(details_link, "📑")