feat: adapt the utils in app.py

app.py

@@ -1,5 +1,4 @@
 import gradio as gr
-import pandas as pd
 from apscheduler.schedulers.background import BackgroundScheduler
 from huggingface_hub import snapshot_download
 
@@ -10,18 +9,15 @@ from src.about import (
 )
 from src.display.css_html_js import custom_css
 from src.display.utils import (
-    BENCHMARK_COLS,
+    QA_BENCHMARK_COLS,
     COLS,
-    EVAL_COLS,
-    NUMERIC_INTERVALS,
     TYPES,
-    AutoEvalColumn,
-    ModelType,
-    fields,
-    Precision
+    AutoEvalColumnQA,
+    fields
 )
 from src.envs import API, EVAL_REQUESTS_PATH, EVAL_RESULTS_PATH, QUEUE_REPO, REPO_ID, RESULTS_REPO, TOKEN
-from src.populate import get_evaluation_queue_df, get_leaderboard_df
+from src.populate import get_leaderboard_df
+from utils import update_table
 
 
 def restart_space():
@@ -45,9 +41,9 @@ try:
 except Exception:
     restart_space()
 
-raw_data, original_df = get_leaderboard_df(
-    EVAL_RESULTS_PATH, EVAL_REQUESTS_PATH, COLS, BENCHMARK_COLS)
-leaderboard_df = original_df.copy()
+raw_data_qa, original_df_qa = get_leaderboard_df(
+    EVAL_RESULTS_PATH, EVAL_REQUESTS_PATH, COLS, QA_BENCHMARK_COLS, task='qa', metric='ndcg_at_1')
+leaderboard_df = original_df_qa.copy()
 
 # (
 #     finished_eval_queue_df,
@@ -56,78 +52,6 @@ leaderboard_df = original_df.copy()
 # ) = get_evaluation_queue_df(EVAL_REQUESTS_PATH, EVAL_COLS)
 
 
-# Searching and filtering
-def update_table(
-        hidden_df: pd.DataFrame,
-        columns: list,
-        type_query: list,
-        precision_query: str,
-        size_query: list,
-        show_deleted: bool,
-        query: str,
-):
-    filtered_df = filter_models(hidden_df, type_query, size_query, precision_query, show_deleted)
-    filtered_df = filter_queries(query, filtered_df)
-    df = select_columns(filtered_df, columns)
-    return df
-
-
-def search_table(df: pd.DataFrame, query: str) -> pd.DataFrame:
-    return df[(df[AutoEvalColumn.model.name].str.contains(query, case=False))]
-
-
-def select_columns(df: pd.DataFrame, columns: list) -> pd.DataFrame:
-    always_here_cols = [
-        AutoEvalColumn.model_type_symbol.name,
-        AutoEvalColumn.model.name,
-    ]
-    # We use COLS to maintain sorting
-    filtered_df = df[
-        always_here_cols + [c for c in COLS if c in df.columns and c in columns]
-        ]
-    return filtered_df
-
-
-def filter_queries(query: str, filtered_df: pd.DataFrame) -> pd.DataFrame:
-    final_df = []
-    if query != "":
-        queries = [q.strip() for q in query.split(";")]
-        for _q in queries:
-            _q = _q.strip()
-            if _q != "":
-                temp_filtered_df = search_table(filtered_df, _q)
-                if len(temp_filtered_df) > 0:
-                    final_df.append(temp_filtered_df)
-        if len(final_df) > 0:
-            filtered_df = pd.concat(final_df)
-            filtered_df = filtered_df.drop_duplicates(
-                subset=[AutoEvalColumn.model.name, AutoEvalColumn.precision.name, AutoEvalColumn.revision.name]
-            )
-
-    return filtered_df
-
-
-def filter_models(
-        df: pd.DataFrame, type_query: list, size_query: list, precision_query: list, show_deleted: bool
-) -> pd.DataFrame:
-    # Show all models
-    if show_deleted:
-        filtered_df = df
-    else:  # Show only still on the hub models
-        filtered_df = df[df[AutoEvalColumn.still_on_hub.name] == True]
-
-    type_emoji = [t[0] for t in type_query]
-    filtered_df = filtered_df.loc[df[AutoEvalColumn.model_type_symbol.name].isin(type_emoji)]
-    filtered_df = filtered_df.loc[df[AutoEvalColumn.precision.name].isin(precision_query + ["None"])]
-
-    numeric_interval = pd.IntervalIndex(sorted([NUMERIC_INTERVALS[s] for s in size_query]))
-    params_column = pd.to_numeric(df[AutoEvalColumn.params.name], errors="coerce")
-    mask = params_column.apply(lambda x: any(numeric_interval.contains(x)))
-    filtered_df = filtered_df.loc[mask]
-
-    return filtered_df
-
-
 demo = gr.Blocks(css=custom_css)
 with demo:
     gr.HTML(TITLE)
@@ -147,12 +71,12 @@ with demo:
                         shown_columns = gr.CheckboxGroup(
                             choices=[
                                 c.name
-                                for c in fields(AutoEvalColumn)
+                                for c in fields(AutoEvalColumnQA)
                                 if not c.hidden and not c.never_hidden
                             ],
                             value=[
                                 c.name
-                                for c in fields(AutoEvalColumn)
+                                for c in fields(AutoEvalColumnQA)
                                 if c.displayed_by_default and not c.hidden and not c.never_hidden
                             ],
                             label="Select columns to show",
@@ -189,10 +113,10 @@ with demo:
 
             leaderboard_table = gr.components.Dataframe(
                 value=leaderboard_df[
-                    [c.name for c in fields(AutoEvalColumn) if c.never_hidden]
+                    [c.name for c in fields(AutoEvalColumnQA) if c.never_hidden]
                     + shown_columns.value
                     ],
-                headers=[c.name for c in fields(AutoEvalColumn) if c.never_hidden] + shown_columns.value,
+                headers=[c.name for c in fields(AutoEvalColumnQA) if c.never_hidden] + shown_columns.value,
                 datatype=TYPES,
                 elem_id="leaderboard-table",
                 interactive=False,
@@ -201,7 +125,7 @@ with demo:
 
             # Dummy leaderboard for handling the case when the user uses backspace key
             hidden_leaderboard_table_for_search = gr.components.Dataframe(
-                value=original_df[COLS],
+                value=original_df_qa[COLS],
                 headers=COLS,
                 datatype=TYPES,
                 visible=False,

src/benchmarks.py

@@ -106,9 +106,12 @@ metric_list = [
 
 @dataclass
 class Benchmark:
-    name: str  # [task]_[domain]_[language]_[metric], task_key in the json file,
+    name: str  # [domain]_[language]_[metric], task_key in the json file,
     metric: str  # ndcg_at_1 ,metric_key in the json file
     col_name: str  # [domain]_[language], name to display in the leaderboard
+    domain: str
+    lang: str
+    task: str
 
 qa_benchmark_dict = {}
 long_doc_benchmark_dict = {}
@@ -116,18 +119,20 @@ for task, domain_dict in dataset_dict.items():
     for domain, lang_dict in domain_dict.items():
         for lang, dataset_list in lang_dict.items():
             if task == "qa":
-                benchmark_name = f"{task}_{domain}_{lang}"
+                benchmark_name = f"{domain}_{lang}"
                 benchmark_name = get_safe_name(benchmark_name)
                 col_name = f"{domain}_{lang}"
                 for metric in dataset_list:
-                    qa_benchmark_dict[benchmark_name] = Benchmark(benchmark_name, metric, col_name)
+                    qa_benchmark_dict[benchmark_name] = Benchmark(benchmark_name, metric, col_name, domain, lang, task)
             elif task == "long_doc":
                 for dataset in dataset_list:
                     col_name = f"{domain}_{lang}_{dataset}"
                     for metric in metric_list:
-                        benchmark_name = f"{task}_{domain}_{lang}_{dataset}_{metric}"
+                        benchmark_name = f"{domain}_{lang}_{dataset}_{metric}"
                         benchmark_name = get_safe_name(benchmark_name)
-                        long_doc_benchmark_dict[benchmark_name] = Benchmark(benchmark_name, metric, col_name)
+                        long_doc_benchmark_dict[benchmark_name] = Benchmark(benchmark_name, metric, col_name, domain, lang, task)
 
 BenchmarksQA = Enum('BenchmarksQA', qa_benchmark_dict)
 BenchmarksLongDoc = Enum('BenchmarksLongDoc', long_doc_benchmark_dict)
+
+BENCHMARK_COLS_QA = [c.col_name for c in qa_benchmark_dict.values()]

src/populate.py

@@ -9,16 +9,17 @@ from src.leaderboard.read_evals import get_raw_eval_results, EvalResult
 from typing import Tuple
 
 
-def get_leaderboard_df(results_path: str, requests_path: str, cols: list, benchmark_cols: list) -> Tuple[list[EvalResult], pd.DataFrame]:
+def get_leaderboard_df(results_path: str, requests_path: str, cols: list, benchmark_cols: list, task: str, metric: str) -> Tuple[list[EvalResult], pd.DataFrame]:
     """Creates a dataframe from all the individual experiment results"""
     raw_data = get_raw_eval_results(results_path, requests_path)
     all_data_json = []
     for v in raw_data:
-        all_data_json += v.to_dict()
+        all_data_json += v.to_dict(task=task, metric=metric)
 
     df = pd.DataFrame.from_records(all_data_json)
-    df["Average ⬆️"] = df[benchmark_cols].mean(axis=1)
-    # df = df.sort_values(by=[AutoEvalColumnQA.average.name], ascending=False)
+    df[AutoEvalColumnQA.average.name] = df[benchmark_cols].mean(axis=1)
+    df = df.sort_values(by=[AutoEvalColumnQA.average.name], ascending=False)
+    df.reset_index(inplace=True)
     df = df[cols].round(decimals=2)
 
     # filter out if any of the benchmarks have not been produced

tests/src/test_populate.py

@@ -11,10 +11,13 @@ def test_get_leaderboard_df():
     benchmark_cols = ['wiki_en', 'wiki_zh',]
     raw_data, df = get_leaderboard_df(results_path, requests_path, cols, benchmark_cols)
     assert df.shape[0] == 2
-    assert df["Retrieval Model"][0] == "bge-m3"
-    assert df["Retrieval Model"][1] == "bge-m3"
-    assert df["Reranking Model"][0] == "NoReranker"
-    assert df["Reranking Model"][1] == "bge-reranker-v2-m3"
+    # the results contains only one embedding model
+    for i in range(2):
+        assert df["Retrieval Model"][i] == "bge-m3"
+    # the results contains only two reranking model
+    assert df["Reranking Model"][0] == "bge-reranker-v2-m3"
+    assert df["Reranking Model"][1] == "NoReranker"
+    assert df["Average ⬆️"][0] > df["Average ⬆️"][1]
     assert not df[['Average ⬆️', 'wiki_en', 'wiki_zh',]].isnull().values.any()
 
 

tests/test_utils.py

@@ -0,0 +1,53 @@
+import pandas as pd
+import pytest
+
+from utils import filter_models, search_table, filter_queries, select_columns
+
+
+@pytest.fixture
+def toy_df():
+    return pd.DataFrame(
+        {
+            "Retrieval Model": [
+                "bge-m3",
+                "bge-m3",
+                "jina-embeddings-v2-base",
+                "jina-embeddings-v2-base"
+            ],
+            "Reranking Model": [
+                "bge-reranker-v2-m3",
+                "NoReranker",
+                "bge-reranker-v2-m3",
+                "NoReranker"
+            ],
+            "Average ⬆️": [0.6, 0.4, 0.3, 0.2],
+            "wiki_en": [0.8, 0.7, 0.2, 0.1],
+            "wiki_zh": [0.4, 0.1, 0.4, 0.3],
+            "news_en": [0.8, 0.7, 0.2, 0.1],
+            "news_zh": [0.4, 0.1, 0.4, 0.3],
+        }
+    )
+
+
+def test_filter_models(toy_df):
+    df_result = filter_models(toy_df, ["bge-reranker-v2-m3", ])
+    assert len(df_result) == 2
+    assert df_result.iloc[0]["Reranking Model"] == "bge-reranker-v2-m3"
+
+
+def test_search_table(toy_df):
+    df_result = search_table(toy_df, "jina")
+    assert len(df_result) == 2
+    assert df_result.iloc[0]["Retrieval Model"] == "jina-embeddings-v2-base"
+
+
+def test_filter_queries(toy_df):
+    df_result = filter_queries("jina", toy_df)
+    assert len(df_result) == 2
+    assert df_result.iloc[0]["Retrieval Model"] == "jina-embeddings-v2-base"
+
+
+def test_select_columns(toy_df):
+    df_result = select_columns(toy_df, ['news',], ['zh',])
+    assert len(df_result.columns) == 4
+    assert df_result['Average ⬆️'].equals(df_result['news_zh'])

tests/toydata/test_results/bge-m3/NoReranker/results_demo_2023-12-21T18-10-08.json

@@ -27,7 +27,7 @@
         "domain": "wiki",
         "lang": "en",
         "dataset": "unknown",
-        "value": 0.69083
+        "value": 0.39083
       }
     ]
   },

utils.py

@@ -0,0 +1,70 @@
+import pandas as pd
+
+from src.display.utils import AutoEvalColumnQA, COLS
+from src.benchmarks import BENCHMARK_COLS_QA, BenchmarksQA
+
+
+def filter_models(df: pd.DataFrame, reranking_query: list) -> pd.DataFrame:
+    return df.loc[df["Reranking Model"].isin(reranking_query)]
+
+
+def filter_queries(query: str, filtered_df: pd.DataFrame) -> pd.DataFrame:
+    final_df = []
+    if query != "":
+        queries = [q.strip() for q in query.split(";")]
+        for _q in queries:
+            _q = _q.strip()
+            if _q != "":
+                temp_filtered_df = search_table(filtered_df, _q)
+                if len(temp_filtered_df) > 0:
+                    final_df.append(temp_filtered_df)
+        if len(final_df) > 0:
+            filtered_df = pd.concat(final_df)
+            filtered_df = filtered_df.drop_duplicates(
+                subset=[
+                    AutoEvalColumnQA.retrieval_model.name,
+                    AutoEvalColumnQA.reranking_model.name,
+                ]
+            )
+
+    return filtered_df
+
+
+def search_table(df: pd.DataFrame, query: str) -> pd.DataFrame:
+    return df[(df[AutoEvalColumnQA.retrieval_model.name].str.contains(query, case=False))]
+
+
+def select_columns(df: pd.DataFrame, domain_query: list, language_query: list) -> pd.DataFrame:
+    always_here_cols = [
+        AutoEvalColumnQA.retrieval_model.name,
+        AutoEvalColumnQA.reranking_model.name,
+        AutoEvalColumnQA.average.name
+    ]
+    selected_cols = []
+    for c in COLS:
+        if c not in df.columns:
+            continue
+        if c not in BENCHMARK_COLS_QA:
+            continue
+        eval_col = BenchmarksQA[c].value
+        if eval_col.domain not in domain_query:
+            continue
+        if eval_col.lang not in language_query:
+            continue
+        selected_cols.append(c)
+    # We use COLS to maintain sorting
+    filtered_df = df[always_here_cols + selected_cols]
+    filtered_df[AutoEvalColumnQA.average.name] = filtered_df[selected_cols].mean(axis=1)
+    return filtered_df
+
+
+def update_table(
+        hidden_df: pd.DataFrame,
+        columns: list,
+        reranking_query: list,
+        query: str,
+):
+    filtered_df = filter_models(hidden_df, reranking_query)
+    filtered_df = filter_queries(query, filtered_df)
+    df = select_columns(filtered_df, columns)
+    return df