refactor: remove the legacy imports

app.py

@@ -87,7 +87,6 @@ def update_metric_long_doc(
 
 
 def update_datastore(version):
-    print("triggered update_datastore")
     global datastore
     global data
     datastore = data[version]
@@ -104,7 +103,6 @@ def update_datastore(version):
 def update_datastore_long_doc(version):
     global datastore
     global data
-    print("triggered update_datastore_long_doc")
     datastore = data[version]
     selected_domains = get_domain_dropdown(LongDocBenchmarks[datastore.slug])
     selected_langs = get_language_dropdown(LongDocBenchmarks[datastore.slug])
@@ -336,12 +334,11 @@ with demo:
                             show_anonymous = get_anonymous_checkbox()
                         with gr.Row():
                             show_revision_and_timestamp = get_revision_and_ts_checkbox()
-                with gr.Tabs(elem_classes="tab-buttons") as sub_tabs:
+                with gr.Tabs(elem_classes="tab-buttons"):
                     with gr.TabItem("Retrieval + Reranking", id=20):
                         with gr.Row():
                             with gr.Column():
                                 search_bar = get_search_bar()
-                            # select reranking model
                             with gr.Column():
                                 selected_rerankings = get_reranking_dropdown(datastore.reranking_models)
 

src/benchmarks.py

@@ -3,7 +3,7 @@ from enum import Enum
 
 from air_benchmark.tasks.tasks import BenchmarkTable
 
-from src.envs import METRIC_LIST
+from src.envs import METRIC_LIST, BENCHMARK_VERSION_LIST
 
 
 def get_safe_name(name: str):
@@ -59,19 +59,16 @@ def get_benchmarks_enum(benchmark_version, task_type):
     return benchmark_dict
 
 
-versions = ("AIR-Bench_24.04", "AIR-Bench_24.05")
 qa_benchmark_dict = {}
-for version in versions:
+for version in BENCHMARK_VERSION_LIST:
     safe_version_name = get_safe_name(version)[-4:]
     qa_benchmark_dict[safe_version_name] = Enum(f"QABenchmarks_{safe_version_name}", get_benchmarks_enum(version, "qa"))
 
 long_doc_benchmark_dict = {}
-for version in versions:
+for version in BENCHMARK_VERSION_LIST:
     safe_version_name = get_safe_name(version)[-4:]
     long_doc_benchmark_dict[safe_version_name] = Enum(f"LongDocBenchmarks_{safe_version_name}", get_benchmarks_enum(version, "long-doc"))
 
-# _qa_benchmark_dict, = get_benchmarks_enum('AIR-Bench_24.04', "qa")
-# _long_doc_benchmark_dict = get_benchmarks_enum('AIR-Bench_24.04', "long-doc")
 
 QABenchmarks = Enum('QABenchmarks', qa_benchmark_dict)
 LongDocBenchmarks = Enum('LongDocBenchmarks', long_doc_benchmark_dict)

src/display/columns.py

@@ -1,6 +1,5 @@
 from dataclasses import dataclass, make_dataclass
 
-from src.benchmarks import QABenchmarks, LongDocBenchmarks
 from src.envs import COL_NAME_AVG, COL_NAME_RETRIEVAL_MODEL, COL_NAME_RERANKING_MODEL, COL_NAME_RETRIEVAL_MODEL_LINK, \
     COL_NAME_RERANKING_MODEL_LINK, COL_NAME_RANK, COL_NAME_REVISION, COL_NAME_TIMESTAMP, COL_NAME_IS_ANONYMOUS
 
@@ -76,22 +75,7 @@ def get_default_col_names_and_types(benchmarks):
     col_types = [c.type for c in fields(AutoEvalColumn) if not c.hidden]
     return col_names, col_types
 
-# AutoEvalColumnQA = make_autoevalcolumn("AutoEvalColumnQA", QABenchmarks)
-# COLS_QA = [c.name for c in fields(AutoEvalColumnQA) if not c.hidden]
-# TYPES_QA = [c.type for c in fields(AutoEvalColumnQA) if not c.hidden]
-
 
 def get_fixed_col_names_and_types():
     fixed_cols = get_default_auto_eval_column_dict()[:-3]
     return [c.name for _, _, c in fixed_cols], [c.type for _, _, c in fixed_cols]
-
-# fixed_cols = get_default_auto_eval_column_dict()[:-3]
-# FIXED_COLS = [c.name for _, _, c in fixed_cols]
-# FIXED_COLS_TYPES = [c.type for _, _, c in fixed_cols]
-
-
-# AutoEvalColumnLongDoc = make_autoevalcolumn("AutoEvalColumnLongDoc", LongDocBenchmarks)
-# COLS_LONG_DOC = [c.name for c in fields(AutoEvalColumnLongDoc) if not c.hidden]
-# TYPES_LONG_DOC = [c.type for c in fields(AutoEvalColumnLongDoc) if not c.hidden]
-
-# Column selection

src/display/components.py

@@ -1,6 +1,7 @@
 import gradio as gr
+
 from src.envs import BENCHMARK_VERSION_LIST, LATEST_BENCHMARK_VERSION
-from src.benchmarks import QABenchmarks
+
 
 def get_version_dropdown():
     return gr.Dropdown(

src/envs.py

@@ -27,7 +27,7 @@ BM25_LINK = model_hyperlink("https://github.com/castorini/pyserini", "BM25")
 
 BENCHMARK_VERSION_LIST = [
     "AIR-Bench_24.04",
-    # "AIR-Bench_24.05",
+    "AIR-Bench_24.05",
 ]
 
 LATEST_BENCHMARK_VERSION = BENCHMARK_VERSION_LIST[0]

src/loaders.py

@@ -5,7 +5,6 @@ import pandas as pd
 
 from src.envs import DEFAULT_METRIC_QA, DEFAULT_METRIC_LONG_DOC, COL_NAME_REVISION, COL_NAME_TIMESTAMP, \
     COL_NAME_IS_ANONYMOUS, BENCHMARK_VERSION_LIST
-
 from src.models import FullEvalResult, LeaderboardDataStore
 from src.utils import get_default_cols, get_leaderboard_df
 
@@ -50,6 +49,7 @@ def load_raw_eval_results(results_path: str) -> List[FullEvalResult]:
             continue
     return results
 
+
 def get_safe_name(name: str):
     """Get RFC 1123 compatible safe name"""
     name = name.replace('-', '_')
@@ -58,6 +58,7 @@ def get_safe_name(name: str):
         for character in name
         if (character.isalnum() or character == '_'))
 
+
 def load_leaderboard_datastore(file_path, version) -> LeaderboardDataStore:
     slug = get_safe_name(version)[-4:]
     lb_data_store = LeaderboardDataStore(version, slug, None, None, None, None, None, None, None, None)
@@ -69,8 +70,6 @@ def load_leaderboard_datastore(file_path, version) -> LeaderboardDataStore:
     print(f'QA data loaded: {lb_data_store.raw_df_qa.shape}')
     lb_data_store.leaderboard_df_qa = lb_data_store.raw_df_qa.copy()
     shown_columns_qa, types_qa = get_default_cols('qa', lb_data_store.slug, add_fix_cols=True)
-    # shown_columns_qa, types_qa = get_default_cols(
-    #     'qa', lb_data_store.leaderboard_df_qa.columns, add_fix_cols=True)
     lb_data_store.types_qa = types_qa
     lb_data_store.leaderboard_df_qa = \
         lb_data_store.leaderboard_df_qa[~lb_data_store.leaderboard_df_qa[COL_NAME_IS_ANONYMOUS]][shown_columns_qa]
@@ -95,7 +94,6 @@ def load_leaderboard_datastore(file_path, version) -> LeaderboardDataStore:
 
 def load_eval_results(file_path: str):
     output = {}
-    # versions = BENCHMARK_VERSION_LIST
     for version in BENCHMARK_VERSION_LIST:
         fn = f"{file_path}/{version}"
         output[version] = load_leaderboard_datastore(fn, version)

src/models.py

@@ -6,9 +6,9 @@ from typing import List, Optional
 import pandas as pd
 
 from src.benchmarks import get_safe_name
+from src.display.formatting import make_clickable_model
 from src.envs import COL_NAME_RETRIEVAL_MODEL, COL_NAME_RERANKING_MODEL, COL_NAME_RETRIEVAL_MODEL_LINK, \
     COL_NAME_RERANKING_MODEL_LINK, COL_NAME_REVISION, COL_NAME_TIMESTAMP, COL_NAME_IS_ANONYMOUS
-from src.display.formatting import make_clickable_model
 
 
 @dataclass
@@ -92,7 +92,8 @@ class FullEvalResult:
 
     def to_dict(self, task='qa', metric='ndcg_at_3') -> List:
         """
-        Convert the results in all the EvalResults over different tasks and metrics. The output is a list of dict compatible with the dataframe UI
+        Convert the results in all the EvalResults over different tasks and metrics.
+        The output is a list of dict compatible with the dataframe UI
         """
         results = defaultdict(dict)
         for eval_result in self.results:
@@ -111,7 +112,6 @@ class FullEvalResult:
             results[eval_result.eval_name][COL_NAME_TIMESTAMP] = self.timestamp
             results[eval_result.eval_name][COL_NAME_IS_ANONYMOUS] = self.is_anonymous
 
-            # print(f'result loaded: {eval_result.eval_name}')
             for result in eval_result.results:
                 # add result for each domain, language, and dataset
                 domain = result["domain"]

src/utils.py

@@ -1,18 +1,17 @@
-import json
 import hashlib
+import json
+import re
 from datetime import datetime, timezone
 from pathlib import Path
 
 import pandas as pd
 
 from src.benchmarks import QABenchmarks, LongDocBenchmarks
-from src.display.formatting import styled_message, styled_error
 from src.display.columns import get_default_col_names_and_types, get_fixed_col_names_and_types
+from src.display.formatting import styled_message, styled_error
 from src.envs import API, SEARCH_RESULTS_REPO, LATEST_BENCHMARK_VERSION, COL_NAME_AVG, COL_NAME_RETRIEVAL_MODEL, \
     COL_NAME_RERANKING_MODEL, COL_NAME_RANK, COL_NAME_REVISION, COL_NAME_TIMESTAMP, COL_NAME_IS_ANONYMOUS
 
-import re
-
 
 def calculate_mean(row):
     if pd.isna(row).any():
@@ -20,6 +19,7 @@ def calculate_mean(row):
     else:
         return row.mean()
 
+
 def remove_html(input_str):
     # Regular expression for finding HTML tags
     clean = re.sub(r'<.*?>', '', input_str)
@@ -59,7 +59,7 @@ def search_table(df: pd.DataFrame, query: str) -> pd.DataFrame:
     return df[(df[COL_NAME_RETRIEVAL_MODEL].str.contains(query, case=False))]
 
 
-def get_default_cols(task: str, version_slug, add_fix_cols: bool=True) -> tuple:
+def get_default_cols(task: str, version_slug, add_fix_cols: bool = True) -> tuple:
     cols = []
     types = []
     if task == "qa":
@@ -105,6 +105,8 @@ def select_columns(
             eval_col = QABenchmarks[version_slug].value[c].value
         elif task == "long-doc":
             eval_col = LongDocBenchmarks[version_slug].value[c].value
+        else:
+            raise NotImplemented
         if eval_col.domain not in domain_query:
             continue
         if eval_col.lang not in language_query:
@@ -122,6 +124,7 @@ def select_columns(
 
     return filtered_df
 
+
 def get_safe_name(name: str):
     """Get RFC 1123 compatible safe name"""
     name = name.replace('-', '_')
@@ -130,6 +133,7 @@ def get_safe_name(name: str):
         for character in name
         if (character.isalnum() or character == '_'))
 
+
 def _update_table(
         task: str,
         version: str,
@@ -249,9 +253,9 @@ def submit_results(
         filepath: str,
         model: str,
         model_url: str,
-        reranking_model: str="",
-        reranking_model_url: str="",
-        version: str=LATEST_BENCHMARK_VERSION,
+        reranking_model: str = "",
+        reranking_model_url: str = "",
+        version: str = LATEST_BENCHMARK_VERSION,
         is_anonymous=False):
     if not filepath.endswith(".zip"):
         return styled_error(f"file uploading aborted. wrong file type: {filepath}")
@@ -280,7 +284,7 @@ def submit_results(
 
     if not reranking_model:
         reranking_model = 'NoReranker'
-    
+
     API.upload_file(
         path_or_fileobj=filepath,
         path_in_repo=f"{version}/{model}/{reranking_model}/{output_fn}",
@@ -384,14 +388,15 @@ def set_listeners(
         search_bar,
         show_anonymous
     ]
-    search_bar_args = [source_df, version,] + selector_list
-    selector_args = [version, source_df] + selector_list + [show_revision_and_timestamp,]
+    search_bar_args = [source_df, version, ] + selector_list
+    selector_args = [version, source_df] + selector_list + [show_revision_and_timestamp, ]
     # Set search_bar listener
     search_bar.submit(update_table_func, search_bar_args, target_df)
 
     # Set column-wise listener
     for selector in selector_list:
-        selector.change(update_table_func, selector_args, target_df, queue=True,)
+        selector.change(update_table_func, selector_args, target_df, queue=True, )
+
 
 def update_table(
         version: str,