feat: add revision and timestamp information

app.py

@@ -8,18 +8,18 @@ from src.about import (
     TITLE,
     EVALUATION_QUEUE_TEXT
 )
+from src.benchmarks import DOMAIN_COLS_QA, LANG_COLS_QA, DOMAIN_COLS_LONG_DOC, LANG_COLS_LONG_DOC, METRIC_LIST, \
+    DEFAULT_METRIC
 from src.display.css_html_js import custom_css
-from src.leaderboard.read_evals import get_raw_eval_results, get_leaderboard_df
-
 from src.envs import API, EVAL_RESULTS_PATH, REPO_ID, RESULTS_REPO, TOKEN
+from src.leaderboard.read_evals import get_raw_eval_results, get_leaderboard_df
 from utils import update_table, update_metric, update_table_long_doc, upload_file, get_default_cols, submit_results
-from src.benchmarks import DOMAIN_COLS_QA, LANG_COLS_QA, DOMAIN_COLS_LONG_DOC, LANG_COLS_LONG_DOC, METRIC_LIST, DEFAULT_METRIC
-from src.display.utils import TYPES_QA, TYPES_LONG_DOC
 
 
 def restart_space():
     API.restart_space(repo_id=REPO_ID)
 
+
 try:
     snapshot_download(
         repo_id=RESULTS_REPO, local_dir=EVAL_RESULTS_PATH, repo_type="dataset", tqdm_class=None, etag_timeout=30,
@@ -39,11 +39,12 @@ print(f'QA data loaded: {original_df_qa.shape}')
 print(f'Long-Doc data loaded: {len(original_df_long_doc)}')
 
 leaderboard_df_qa = original_df_qa.copy()
-shown_columns_qa = get_default_cols('qa', leaderboard_df_qa.columns, add_fix_cols=True)
+shown_columns_qa, types_qa = get_default_cols('qa', leaderboard_df_qa.columns, add_fix_cols=True)
 leaderboard_df_qa = leaderboard_df_qa[shown_columns_qa]
 
 leaderboard_df_long_doc = original_df_long_doc.copy()
-shown_columns_long_doc = get_default_cols('long-doc', leaderboard_df_long_doc.columns, add_fix_cols=True)
+shown_columns_long_doc, types_long_doc = get_default_cols('long-doc', leaderboard_df_long_doc.columns,
+                                                          add_fix_cols=True)
 leaderboard_df_long_doc = leaderboard_df_long_doc[shown_columns_long_doc]
 
 
@@ -56,6 +57,7 @@ def update_metric_qa(
 ):
     return update_metric(raw_data, 'qa', metric, domains, langs, reranking_model, query)
 
+
 def update_metric_long_doc(
         metric: str,
         domains: list,
@@ -124,7 +126,7 @@ with demo:
 
             leaderboard_table = gr.components.Dataframe(
                 value=leaderboard_df_qa,
-                datatype=TYPES_QA,
+                datatype=types_qa,
                 elem_id="leaderboard-table",
                 interactive=False,
                 visible=True,
@@ -133,7 +135,7 @@ with demo:
             # Dummy leaderboard for handling the case when the user uses backspace key
             hidden_leaderboard_table_for_search = gr.components.Dataframe(
                 value=leaderboard_df_qa,
-                datatype=TYPES_QA,
+                datatype=types_qa,
                 # headers=COLS,
                 # datatype=TYPES,
                 visible=False,
@@ -234,7 +236,7 @@ with demo:
 
             leaderboard_table_long_doc = gr.components.Dataframe(
                 value=leaderboard_df_long_doc,
-                datatype=TYPES_LONG_DOC,
+                datatype=types_long_doc,
                 elem_id="leaderboard-table-long-doc",
                 interactive=False,
                 visible=True,
@@ -243,7 +245,7 @@ with demo:
             # Dummy leaderboard for handling the case when the user uses backspace key
             hidden_leaderboard_table_for_search = gr.components.Dataframe(
                 value=leaderboard_df_long_doc,
-                datatype=TYPES_LONG_DOC,
+                datatype=types_long_doc,
                 visible=False,
             )
 
@@ -300,7 +302,7 @@ with demo:
                 with gr.Row():
                     with gr.Column():
                         benchmark_version = gr.Dropdown(
-                            ["AIR-Bench_24.04",],
+                            ["AIR-Bench_24.04", ],
                             value="AIR-Bench_24.04",
                             interactive=True,
                             label="AIR-Bench Version")

src/display/utils.py

@@ -25,29 +25,42 @@ COL_NAME_RERANKING_MODEL = "Reranking Model"
 COL_NAME_RETRIEVAL_MODEL_LINK = "Retrieval Model LINK"
 COL_NAME_RERANKING_MODEL_LINK = "Reranking Model LINK"
 COL_NAME_RANK = "Rank 🏆"
+COL_NAME_REVISION = "Revision"
+COL_NAME_TIMESTAMP = "Submission Date"
 
-def make_autoevalcolumn(cls_name="BenchmarksQA", benchmarks=BenchmarksQA):
-    ## Leaderboard columns
+
+def get_default_auto_eval_column_dict():
     auto_eval_column_dict = []
     # Init
     auto_eval_column_dict.append(
-        ["retrieval_model", ColumnContent, ColumnContent(COL_NAME_RETRIEVAL_MODEL, "markdown", True, never_hidden=True)]
+        ["rank", ColumnContent, ColumnContent(COL_NAME_RANK, "number", True)]
+    )
+    auto_eval_column_dict.append(
+        ["retrieval_model", ColumnContent, ColumnContent(COL_NAME_RETRIEVAL_MODEL, "markdown", True, hidden=False, never_hidden=True)]
     )
     auto_eval_column_dict.append(
-        ["reranking_model", ColumnContent, ColumnContent(COL_NAME_RERANKING_MODEL, "markdown", True, never_hidden=True)]
+        ["reranking_model", ColumnContent, ColumnContent(COL_NAME_RERANKING_MODEL, "markdown", True, hidden=False, never_hidden=True)]
     )
     auto_eval_column_dict.append(
-        ["retrieval_model_link", ColumnContent, ColumnContent(COL_NAME_RETRIEVAL_MODEL, "markdown", False, hidden=True, never_hidden=False)]
+        ["revision", ColumnContent, ColumnContent(COL_NAME_REVISION, "markdown", True, never_hidden=True)]
     )
     auto_eval_column_dict.append(
-        ["reranking_model_link", ColumnContent, ColumnContent(COL_NAME_RERANKING_MODEL, "markdown", False, hidden=True, never_hidden=False)]
+        ["timestamp", ColumnContent, ColumnContent(COL_NAME_TIMESTAMP, "date", True, never_hidden=True)]
     )
     auto_eval_column_dict.append(
         ["average", ColumnContent, ColumnContent(COL_NAME_AVG, "number", True)]
     )
     auto_eval_column_dict.append(
-        ["rank", ColumnContent, ColumnContent(COL_NAME_RANK, "number", True)]
+        ["retrieval_model_link", ColumnContent, ColumnContent(COL_NAME_RETRIEVAL_MODEL, "markdown", False, hidden=True, never_hidden=False)]
+    )
+    auto_eval_column_dict.append(
+        ["reranking_model_link", ColumnContent, ColumnContent(COL_NAME_RERANKING_MODEL, "markdown", False, hidden=True, never_hidden=False)]
     )
+    return auto_eval_column_dict
+
+def make_autoevalcolumn(cls_name="BenchmarksQA", benchmarks=BenchmarksQA):
+    auto_eval_column_dict = get_default_auto_eval_column_dict()
+    ## Leaderboard columns
     for benchmark in benchmarks:
         auto_eval_column_dict.append(
             [benchmark.name, ColumnContent, ColumnContent(benchmark.value.col_name, "number", True)]

src/leaderboard/read_evals.py

@@ -13,6 +13,8 @@ from src.display.utils import (
     COL_NAME_RETRIEVAL_MODEL,
     COL_NAME_RERANKING_MODEL_LINK,
     COL_NAME_RETRIEVAL_MODEL_LINK,
+    COL_NAME_REVISION,
+    COL_NAME_TIMESTAMP,
     COLS_QA,
     QA_BENCHMARK_COLS,
     COLS_LONG_DOC,
@@ -37,6 +39,7 @@ class EvalResult:
     task: str
     metric: str
     timestamp: str = ""  # submission timestamp
+    revision: str = ""
 
 
 @dataclass
@@ -50,7 +53,8 @@ class FullEvalResult:
     retrieval_model_link: str
     reranking_model_link: str
     results: List[EvalResult]  # results on all the EvalResults over different tasks and metrics.
-    date: str = ""
+    timestamp: str = ""
+    revision: str = ""
 
     @classmethod
     def init_from_json_file(cls, json_filepath):
@@ -65,20 +69,25 @@ class FullEvalResult:
         result_list = []
         retrieval_model_link = ""
         reranking_model_link = ""
+        revision = ""
         for item in model_data:
             config = item.get("config", {})
             # eval results for different metrics
             results = item.get("results", [])
-            retrieval_model_link=config["retrieval_model_link"]
-            if config["reranking_model_link"] is not None:
-                reranking_model_link=""
+            retrieval_model_link = config["retrieval_model_link"]
+            if config["reranking_model_link"] is None:
+                reranking_model_link = ""
+            else:
+                reranking_model_link = config["reranking_model_link"]
             eval_result = EvalResult(
                 eval_name=f"{config['retrieval_model']}_{config['reranking_model']}_{config['metric']}",
                 retrieval_model=config["retrieval_model"],
                 reranking_model=config["reranking_model"],
                 results=results,
                 task=config["task"],
-                metric=config["metric"]
+                metric=config["metric"],
+                timestamp=config.get("timestamp", "2024-05-12T12:24:02Z"),
+                revision=config.get("revision", "3a2ba9dcad796a48a02ca1147557724e")
             )
             result_list.append(eval_result)
         return cls(
@@ -87,7 +96,9 @@ class FullEvalResult:
             reranking_model=result_list[0].reranking_model,
             retrieval_model_link=retrieval_model_link,
             reranking_model_link=reranking_model_link,
-            results=result_list
+            results=result_list,
+            timestamp=result_list[0].timestamp,
+            revision=result_list[0].revision
         )
 
     def to_dict(self, task='qa', metric='ndcg_at_3') -> List:
@@ -107,6 +118,8 @@ class FullEvalResult:
                 make_clickable_model(self.reranking_model, self.reranking_model_link))
             results[eval_result.eval_name][COL_NAME_RETRIEVAL_MODEL_LINK] = self.retrieval_model_link
             results[eval_result.eval_name][COL_NAME_RERANKING_MODEL_LINK] = self.reranking_model_link
+            results[eval_result.eval_name][COL_NAME_REVISION] = self.revision
+            results[eval_result.eval_name][COL_NAME_TIMESTAMP] = self.timestamp
 
             # print(f'result loaded: {eval_result.eval_name}')
             for result in eval_result.results:
@@ -193,4 +206,7 @@ def get_leaderboard_df(raw_data: List[FullEvalResult], task: str, metric: str) -
     # filter out if any of the benchmarks have not been produced
     df = df[has_no_nan_values(df, _benchmark_cols)]
     df[COL_NAME_RANK] = df[COL_NAME_AVG].rank(ascending=False, method="min")
+
+    # shorten the revision
+    df[COL_NAME_REVISION] = df[COL_NAME_REVISION].str[:6]
     return df

tests/src/display/test_utils.py

@@ -1,5 +1,5 @@
 import pytest
-from src.display.utils import fields, AutoEvalColumnQA, COLS_QA, COLS_LONG_DOC, COLS_LITE, TYPES, QA_BENCHMARK_COLS, LONG_DOC_BENCHMARK_COLS
+from src.display.utils import fields, AutoEvalColumnQA, COLS_QA, COLS_LONG_DOC, COLS_LITE, TYPES_QA, TYPES_LONG_DOC, QA_BENCHMARK_COLS, LONG_DOC_BENCHMARK_COLS
 
 
 def test_fields():
@@ -11,6 +11,7 @@ def test_macro_variables():
     print(f'COLS_QA: {COLS_QA}')
     print(f'COLS_LONG_DOC: {COLS_LONG_DOC}')
     print(f'COLS_LITE: {COLS_LITE}')
-    print(f'TYPES: {TYPES}')
+    print(f'TYPES_QA: {TYPES_QA}')
+    print(f'TYPES_LONG_DOC: {TYPES_LONG_DOC}')
     print(f'QA_BENCHMARK_COLS: {QA_BENCHMARK_COLS}')
     print(f'LONG_DOC_BENCHMARK_COLS: {LONG_DOC_BENCHMARK_COLS}')

tests/test_utils.py

@@ -1,7 +1,8 @@
 import pandas as pd
 import pytest
 
-from utils import filter_models, search_table, filter_queries, select_columns, update_table_long_doc, get_iso_format_timestamp
+from utils import filter_models, search_table, filter_queries, select_columns, update_table_long_doc, get_iso_format_timestamp, get_default_cols
+from src.display.utils import COLS_QA
 
 
 @pytest.fixture
@@ -86,3 +87,9 @@ def test_get_iso_format_timestamp():
     assert len(timestamp_fn) == 14
     assert len(timestamp_config) == 20
     assert timestamp_config[-1] == "Z"
+
+
+def test_get_default_cols():
+    cols, types = get_default_cols("qa", COLS_QA)
+    for c, t in zip(cols, types):
+        print(f"type({c}): {t}")

utils.py

@@ -7,7 +7,7 @@ from pathlib import Path
 import pandas as pd
 
 from src.benchmarks import BENCHMARK_COLS_QA, BENCHMARK_COLS_LONG_DOC, BenchmarksQA, BenchmarksLongDoc
-from src.display.utils import AutoEvalColumnQA, AutoEvalColumnLongDoc, COLS_QA, COLS_LONG_DOC, COL_NAME_RANK, COL_NAME_AVG, COL_NAME_RERANKING_MODEL, COL_NAME_RETRIEVAL_MODEL
+from src.display.utils import COLS_QA, TYPES_QA, COLS_LONG_DOC, TYPES_LONG_DOC, COL_NAME_RANK, COL_NAME_AVG, COL_NAME_RERANKING_MODEL, COL_NAME_RETRIEVAL_MODEL, COL_NAME_REVISION, COL_NAME_TIMESTAMP, AutoEvalColumnQA, AutoEvalColumnLongDoc, get_default_auto_eval_column_dict
 from src.leaderboard.read_evals import FullEvalResult, get_leaderboard_df
 from src.envs import API, SEARCH_RESULTS_REPO, CACHE_PATH
 from src.display.formatting import styled_message, styled_error
@@ -44,22 +44,37 @@ def search_table(df: pd.DataFrame, query: str) -> pd.DataFrame:
 
 
 def get_default_cols(task: str, columns: list, add_fix_cols: bool=True) -> list:
+    cols = []
+    types = []
     if task == "qa":
-        cols = list(frozenset(COLS_QA).intersection(frozenset(BENCHMARK_COLS_QA)).intersection(frozenset(columns)))
+        cols_list = COLS_QA
+        types_list = TYPES_QA
+        benchmark_list = BENCHMARK_COLS_QA
     elif task == "long-doc":
-        cols = list(frozenset(COLS_LONG_DOC).intersection(frozenset(BENCHMARK_COLS_LONG_DOC)).intersection(frozenset(columns)))
+        cols_list = COLS_LONG_DOC
+        types_list = TYPES_LONG_DOC
+        benchmark_list = BENCHMARK_COLS_LONG_DOC
     else:
         raise NotImplemented
+    for col_name, col_type in zip(cols_list, types_list):
+        if col_name not in benchmark_list:
+            continue
+        if col_name not in columns:
+            continue
+        cols.append(col_name)
+        types.append(col_type)
+
     if add_fix_cols:
         cols = FIXED_COLS + cols
-    return cols
-
-FIXED_COLS = [
-        COL_NAME_RANK,
-        COL_NAME_RETRIEVAL_MODEL,
-        COL_NAME_RERANKING_MODEL,
-        COL_NAME_AVG,
-    ]
+        types = FIXED_COLS_TYPES + types
+    return cols, types
+
+fixed_cols = get_default_auto_eval_column_dict()[:-2]
+
+
+FIXED_COLS = [c.name for _, _, c in fixed_cols]
+FIXED_COLS_TYPES = [c.type for _, _, c in fixed_cols]
+
 
 def select_columns(df: pd.DataFrame, domain_query: list, language_query: list, task: str = "qa") -> pd.DataFrame:
     cols = get_default_cols(task=task, columns=df.columns, add_fix_cols=False)