Change repository names

.idea/.gitignore

@@ -0,0 +1,8 @@
+# Default ignored files
+/shelf/
+/workspace.xml
+# Editor-based HTTP Client requests
+/httpRequests/
+# Datasource local storage ignored files
+/dataSources/
+/dataSources.local.xml

.idea/aws.xml

@@ -0,0 +1,11 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<project version="4">
+  <component name="accountSettings">
+    <option name="activeRegion" value="us-east-1" />
+    <option name="recentlyUsedRegions">
+      <list>
+        <option value="us-east-1" />
+      </list>
+    </option>
+  </component>
+</project>

app.py

@@ -33,6 +33,7 @@ from src.submission.submit import add_new_eval
 def restart_space():
     API.restart_space(repo_id=REPO_ID, token=TOKEN)
 
+
 try:
     print(EVAL_REQUESTS_PATH)
     snapshot_download(
@@ -48,7 +49,6 @@ try:
 except Exception:
     restart_space()
 
-
 raw_data, original_df = get_leaderboard_df(EVAL_RESULTS_PATH, EVAL_REQUESTS_PATH, COLS, BENCHMARK_COLS)
 leaderboard_df = original_df.copy()
 
@@ -61,13 +61,13 @@ leaderboard_df = original_df.copy()
 
 # Searching and filtering
 def update_table(
-    hidden_df: pd.DataFrame,
-    columns: list,
-    type_query: list,
-    precision_query: str,
-    size_query: list,
-    show_deleted: bool,
-    query: str,
+        hidden_df: pd.DataFrame,
+        columns: list,
+        type_query: list,
+        precision_query: str,
+        size_query: list,
+        show_deleted: bool,
+        query: str,
 ):
     filtered_df = filter_models(hidden_df, type_query, size_query, precision_query, show_deleted)
     filtered_df = filter_queries(query, filtered_df)
@@ -87,7 +87,7 @@ def select_columns(df: pd.DataFrame, columns: list) -> pd.DataFrame:
     # We use COLS to maintain sorting
     filtered_df = df[
         always_here_cols + [c for c in COLS if c in df.columns and c in columns] + [AutoEvalColumn.dummy.name]
-    ]
+        ]
     return filtered_df
 
 
@@ -111,7 +111,7 @@ def filter_queries(query: str, filtered_df: pd.DataFrame) -> pd.DataFrame:
 
 
 def filter_models(
-    df: pd.DataFrame, type_query: list, size_query: list, precision_query: list, show_deleted: bool
+        df: pd.DataFrame, type_query: list, size_query: list, precision_query: list, show_deleted: bool
 ) -> pd.DataFrame:
     # Show all models
     if show_deleted:
@@ -167,7 +167,7 @@ with demo:
                             value=False, label="Show gated/private/deleted models", interactive=True
                         )
                 with gr.Column(min_width=320):
-                    #with gr.Box(elem_id="box-filter"):
+                    # with gr.Box(elem_id="box-filter"):
                     filter_columns_type = gr.CheckboxGroup(
                         label="Model types",
                         choices=[t.to_str() for t in ModelType],
@@ -195,13 +195,13 @@ with demo:
                     [c.name for c in fields(AutoEvalColumn) if c.never_hidden]
                     + shown_columns.value
                     + [AutoEvalColumn.dummy.name]
-                ],
+                    ],
                 headers=[c.name for c in fields(AutoEvalColumn) if c.never_hidden] + shown_columns.value,
                 datatype=TYPES,
                 elem_id="leaderboard-table",
                 interactive=False,
                 visible=True,
-                column_widths=["2%", "33%"] 
+                column_widths=["2%", "33%"]
             )
 
             # Dummy leaderboard for handling the case when the user uses backspace key
@@ -224,7 +224,8 @@ with demo:
                 ],
                 leaderboard_table,
             )
-            for selector in [shown_columns, filter_columns_type, filter_columns_precision, filter_columns_size, deleted_models_visibility]:
+            for selector in [shown_columns, filter_columns_type, filter_columns_precision, filter_columns_size,
+                             deleted_models_visibility]:
                 selector.change(
                     update_table,
                     [
@@ -250,8 +251,8 @@ with demo:
 
                 with gr.Column():
                     with gr.Accordion(
-                        f"✅ Finished Evaluations ({len(finished_eval_queue_df)})",
-                        open=False,
+                            f"✅ Finished Evaluations ({len(finished_eval_queue_df)})",
+                            open=False,
                     ):
                         with gr.Row():
                             finished_eval_table = gr.components.Dataframe(
@@ -261,8 +262,8 @@ with demo:
                                 row_count=5,
                             )
                     with gr.Accordion(
-                        f"🔄 Running Evaluation Queue ({len(running_eval_queue_df)})",
-                        open=False,
+                            f"🔄 Running Evaluation Queue ({len(running_eval_queue_df)})",
+                            open=False,
                     ):
                         with gr.Row():
                             running_eval_table = gr.components.Dataframe(
@@ -273,8 +274,8 @@ with demo:
                             )
 
                     with gr.Accordion(
-                        f"⏳ Pending Evaluation Queue ({len(pending_eval_queue_df)})",
-                        open=False,
+                            f"⏳ Pending Evaluation Queue ({len(pending_eval_queue_df)})",
+                            open=False,
                     ):
                         with gr.Row():
                             pending_eval_table = gr.components.Dataframe(

src/display/utils.py

@@ -5,6 +5,7 @@ import pandas as pd
 
 from src.display.about import Tasks
 
+
 def fields(raw_class):
     return [v for k, v in raw_class.__dict__.items() if k[:2] != "__" and k[-2:] != "__"]
 
@@ -21,13 +22,13 @@ class ColumnContent:
     never_hidden: bool = False
     dummy: bool = False
 
+
 ## Leaderboard columns
-auto_eval_column_dict = []
+auto_eval_column_dict = [["model_type_symbol", ColumnContent, ColumnContent("T", "str", True, never_hidden=True)],
+                         ["model", ColumnContent, ColumnContent("Model", "markdown", True, never_hidden=True)],
+                         ["average", ColumnContent, ColumnContent("Average ⬆️", "number", True)]]
 # Init
-auto_eval_column_dict.append(["model_type_symbol", ColumnContent, ColumnContent("T", "str", True, never_hidden=True)])
-auto_eval_column_dict.append(["model", ColumnContent, ColumnContent("Model", "markdown", True, never_hidden=True)])
-#Scores
-auto_eval_column_dict.append(["average", ColumnContent, ColumnContent("Average ⬆️", "number", True)])
+# Scores
 for task in Tasks:
     auto_eval_column_dict.append([task.name, ColumnContent, ColumnContent(task.value.col_name, "number", True)])
 # Model information
@@ -46,7 +47,8 @@ auto_eval_column_dict.append(["dummy", ColumnContent, ColumnContent("model_name_
 # We use make dataclass to dynamically fill the scores from Tasks
 AutoEvalColumn = make_dataclass("AutoEvalColumn", auto_eval_column_dict, frozen=True)
 
-## For the queue columns in the submission tab
+
+# For the queue columns in the submission tab
 @dataclass(frozen=True)
 class EvalQueueColumn:  # Queue column
     model = ColumnContent("model", "markdown", True)
@@ -56,12 +58,13 @@ class EvalQueueColumn:  # Queue column
     weight_type = ColumnContent("weight_type", "str", "Original")
     status = ColumnContent("status", "str", True)
 
-## All the model information that we might need
+
+# All the model information that we might need
 @dataclass
 class ModelDetails:
     name: str
     display_name: str = ""
-    symbol: str = "" # emoji
+    symbol: str = ""  # emoji
 
 
 class ModelType(Enum):
@@ -86,11 +89,13 @@ class ModelType(Enum):
             return ModelType.IFT
         return ModelType.Unknown
 
+
 class WeightType(Enum):
     Adapter = ModelDetails("Adapter")
     Original = ModelDetails("Original")
     Delta = ModelDetails("Delta")
 
+
 class Precision(Enum):
     float16 = ModelDetails("float16")
     bfloat16 = ModelDetails("bfloat16")
@@ -112,6 +117,7 @@ class Precision(Enum):
             return Precision.qt_GPTQ
         return Precision.Unknown
 
+
 # Column selection
 COLS = [c.name for c in fields(AutoEvalColumn) if not c.hidden]
 TYPES = [c.type for c in fields(AutoEvalColumn) if not c.hidden]

src/envs.py

@@ -5,12 +5,12 @@ from huggingface_hub import HfApi
 # clone / pull the lmeh eval data
 TOKEN = os.environ.get("TOKEN", None)
 
-OWNER = "demo-leaderboard"
-REPO_ID = f"{OWNER}/leaderboard"
-QUEUE_REPO = f"{OWNER}/requests"
-RESULTS_REPO = f"{OWNER}/results"
+OWNER = "AI-Secure"
+REPO_ID = f"{OWNER}/llm-trustworthy-leaderboard"
+QUEUE_REPO = f"{OWNER}/llm-trustworthy-leaderboard-requests"
+RESULTS_REPO = f"{OWNER}/llm-trustworthy-leaderboard-results"
 
-CACHE_PATH=os.getenv("HF_HOME", ".")
+CACHE_PATH = os.getenv("HF_HOME", ".")
 
 # Local caches
 EVAL_REQUESTS_PATH = os.path.join(CACHE_PATH, "eval-queue")

src/submission/submit.py

@@ -14,6 +14,7 @@ from src.submission.check_validity import (
 REQUESTED_MODELS = None
 USERS_TO_SUBMISSION_DATES = None
 
+
 def add_new_eval(
     model: str,
     base_model: str,
@@ -45,7 +46,8 @@ def add_new_eval(
 
     # Is the model on the hub?
     if weight_type in ["Delta", "Adapter"]:
-        base_model_on_hub, error, _ = is_model_on_hub(model_name=base_model, revision=revision, token=TOKEN, test_tokenizer=True)
+        base_model_on_hub, error, _ = is_model_on_hub(model_name=base_model, revision=revision, token=TOKEN,
+                                                      test_tokenizer=True)
         if not base_model_on_hub:
             return styled_error(f'Base model "{base_model}" {error}')
 
@@ -114,5 +116,6 @@ def add_new_eval(
     os.remove(out_path)
 
     return styled_message(
-        "Your request has been submitted to the evaluation queue!\nPlease wait for up to an hour for the model to show in the PENDING list."
+        "Your request has been submitted to the evaluation queue!\nPlease wait for up to an hour for the model to "
+        "show in the PENDING list."
     )