chore: clean up the requests related codes

src/display/utils.py

@@ -19,18 +19,22 @@ class ColumnContent:
     never_hidden: bool = False
 
 
+COL_NAME_AVG = "Average ⬆️"
+COL_NAME_RETRIEVAL_MODEL = "Retrieval Model"
+COL_NAME_RERANKING_MODEL = "Reranking Model"
+
 def make_autoevalcolumn(cls_name="BenchmarksQA", benchmarks=BenchmarksQA):
     ## Leaderboard columns
     auto_eval_column_dict = []
     # Init
     auto_eval_column_dict.append(
-        ["retrieval_model", ColumnContent, ColumnContent("Retrieval Model", "markdown", True, never_hidden=True)]
+        ["retrieval_model", ColumnContent, ColumnContent(COL_NAME_RETRIEVAL_MODEL, "markdown", True, never_hidden=True)]
     )
     auto_eval_column_dict.append(
-        ["reranking_model", ColumnContent, ColumnContent("Reranking Model", "markdown", True, never_hidden=True)]
+        ["reranking_model", ColumnContent, ColumnContent(COL_NAME_RERANKING_MODEL, "markdown", True, never_hidden=True)]
     )
     auto_eval_column_dict.append(
-        ["average", ColumnContent, ColumnContent("Average ⬆️", "number", True)]
+        ["average", ColumnContent, ColumnContent(COL_NAME_AVG, "number", True)]
     )
     for benchmark in benchmarks:
         auto_eval_column_dict.append(

src/leaderboard/read_evals.py

@@ -1,24 +1,28 @@
-import glob
-from collections import defaultdict
 import json
 import os.path
+from collections import defaultdict
 from dataclasses import dataclass
 from typing import List
 
 import dateutil.parser._parser
+import pandas as pd
 
-from src.display.utils import AutoEvalColumnQA
 from src.benchmarks import get_safe_name
+from src.display.formatting import has_no_nan_values
+from src.display.utils import COL_NAME_RERANKING_MODEL, COL_NAME_RETRIEVAL_MODEL, COLS_QA, QA_BENCHMARK_COLS, \
+    COLS_LONG_DOC, LONG_DOC_BENCHMARK_COLS, COL_NAME_AVG
 
 
 @dataclass
 class EvalResult:
-    """Full evaluation result of a single embedding model
+    """
+    Evaluation result of a single embedding model with a specific reranking model on benchmarks over different
+    domains, languages, and datasets
     """
     eval_name: str  # name of the evaluation, [retrieval_model]_[reranking_model]_[metric]
     retrieval_model: str
     reranking_model: str
-    results: list  # results on all the benchmarks over different domains, languages, and datasets. Use benchmark.name as the key
+    results: list  # results on all the benchmarks stored as dict
     task: str
     metric: str
     timestamp: str = ""  # submission timestamp
@@ -26,6 +30,9 @@ class EvalResult:
 
 @dataclass
 class FullEvalResult:
+    """
+    Evaluation result of a single embedding model with a specific reranking model on benchmarks over different tasks
+    """
     eval_name: str  # name of the evaluation, [retrieval_model]_[reranking_model]
     retrieval_model: str
     reranking_model: str
@@ -34,7 +41,8 @@ class FullEvalResult:
 
     @classmethod
     def init_from_json_file(cls, json_filepath):
-        """Initiate from the result json file for a single model.
+        """
+        Initiate from the result json file for a single model.
         The json file will be written only when the status is FINISHED.
         """
         with open(json_filepath) as fp:
@@ -63,19 +71,18 @@ class FullEvalResult:
         )
 
     def to_dict(self, task='qa', metric='ndcg_at_3') -> List:
-        """Convert FullEvalResult to a list of dict compatible with our dataframe UI
+        """
+        Convert the results in all the EvalResults over different tasks and metrics. The output is a list of dict compatible with the dataframe UI
         """
         results = defaultdict(dict)
         for eval_result in self.results:
             if eval_result.metric != metric:
-                # print(f'result skipped: {metric} != {eval_result.metric}')
                 continue
             if eval_result.task != task:
-                # print(f'result skipped: {task} != {eval_result.task}')
                 continue
             results[eval_result.eval_name]["eval_name"] = eval_result.eval_name
-            results[eval_result.eval_name][AutoEvalColumnQA.retrieval_model.name] = self.retrieval_model
-            results[eval_result.eval_name][AutoEvalColumnQA.reranking_model.name] = self.reranking_model
+            results[eval_result.eval_name][COL_NAME_RETRIEVAL_MODEL] = self.retrieval_model
+            results[eval_result.eval_name][COL_NAME_RERANKING_MODEL] = self.reranking_model
 
             print(f'result loaded: {eval_result.eval_name}')
             for result in eval_result.results:
@@ -92,43 +99,20 @@ class FullEvalResult:
         return [v for v in results.values()]
 
 
-def get_request_file_for_model(requests_path, retrieval_model_name, reranking_model_name):
-    """
-    Load the request status from a json file
-    """
-    request_files = os.path.join(
-        requests_path,
-        f"{retrieval_model_name}",
-        f"{reranking_model_name}",
-        "eval_request_*.json",
-    )
-    request_files = glob.glob(request_files)
-
-    request_file = ""
-    request_files = sorted(request_files, reverse=True)
-    for tmp_request_file in request_files:
-        with open(tmp_request_file, "r") as f:
-            req_content = json.load(f)
-            if req_content["status"] in ["FINISHED"]:
-                request_file = tmp_request_file
-                break
-    return request_file
-
-
 def get_raw_eval_results(results_path: str) -> List[FullEvalResult]:
     """
     Load the evaluation results from a json file
     """
     model_result_filepaths = []
     for root, dirs, files in os.walk(results_path):
-        if len(files) == 0 or any([not f.endswith(".json") for f in files]):
+        if len(files) == 0:
             continue
         try:
             files.sort(key=lambda x: x.removesuffix(".json").removeprefix("results_")[:-7], reverse=True)
         except dateutil.parser._parser.ParserError:
             files = [files[-1]]
 
-        # select the latest and finished results
+        # select the latest results
         for file in files:
             model_result_filepaths.append(os.path.join(root, file))
 
@@ -136,7 +120,6 @@ def get_raw_eval_results(results_path: str) -> List[FullEvalResult]:
     for model_result_filepath in model_result_filepaths:
         # create evaluation results
         eval_result = FullEvalResult.init_from_json_file(model_result_filepath)
-        model_result_date_str = model_result_filepath.split('/')[-1].removeprefix("results_").removesuffix(".json")
         print(f'file loaded: {model_result_filepath}')
         eval_name = eval_result.eval_name
         eval_results[eval_name] = eval_result
@@ -150,3 +133,35 @@ def get_raw_eval_results(results_path: str) -> List[FullEvalResult]:
             print(f"loading failed: {k}")
             continue
     return results
+
+
+def get_leaderboard_df(raw_data: List[FullEvalResult], task: str, metric: str) -> pd.DataFrame:
+    """
+    Creates a dataframe from all the individual experiment results
+    """
+    if task == "qa":
+        cols = COLS_QA
+        benchmark_cols = QA_BENCHMARK_COLS
+    elif task == "long_doc":
+        cols = COLS_LONG_DOC
+        benchmark_cols = LONG_DOC_BENCHMARK_COLS
+    else:
+        raise NotImplemented
+    all_data_json = []
+    for v in raw_data:
+        all_data_json += v.to_dict(task=task, metric=metric)
+    df = pd.DataFrame.from_records(all_data_json)
+    print(f'dataframe created: {df.shape}')
+
+    # calculate the average score for selected benchmarks
+    _benchmark_cols = frozenset(benchmark_cols).intersection(frozenset(df.columns.to_list()))
+    df[COL_NAME_AVG] = df[list(_benchmark_cols)].mean(axis=1).round(decimals=2)
+    df = df.sort_values(by=[COL_NAME_AVG], ascending=False)
+    df.reset_index(inplace=True)
+
+    _cols = frozenset(cols).intersection(frozenset(df.columns.to_list()))
+    df = df[_cols].round(decimals=2)
+
+    # filter out if any of the benchmarks have not been produced
+    df = df[has_no_nan_values(df, _benchmark_cols)]
+    return df

src/populate.py

@@ -1,94 +0,0 @@
-import json
-import os
-
-import pandas as pd
-
-from src.display.formatting import has_no_nan_values, make_clickable_model
-from src.display.utils import AutoEvalColumnQA, AutoEvalColumnLongDoc, EvalQueueColumn
-from src.leaderboard.read_evals import get_raw_eval_results, EvalResult, FullEvalResult
-from typing import Tuple, List
-
-
-def get_leaderboard_df(raw_data: List[FullEvalResult], cols: list, benchmark_cols: list, task: str, metric: str) -> pd.DataFrame:
-    """Creates a dataframe from all the individual experiment results"""
-    all_data_json = []
-    for v in raw_data:
-        all_data_json += v.to_dict(task=task, metric=metric)
-    df = pd.DataFrame.from_records(all_data_json)
-    print(f'dataframe created: {df.shape}')
-
-    # calculate the average score for selected benchmarks
-    _benchmark_cols = frozenset(benchmark_cols).intersection(frozenset(df.columns.to_list()))
-    if task == 'qa':
-        df[AutoEvalColumnQA.average.name] = df[list(_benchmark_cols)].mean(axis=1).round(decimals=2)
-        df = df.sort_values(by=[AutoEvalColumnQA.average.name], ascending=False)
-    elif task == "long_doc":
-        df[AutoEvalColumnLongDoc.average.name] = df[list(_benchmark_cols)].mean(axis=1).round(decimals=2)
-        df = df.sort_values(by=[AutoEvalColumnLongDoc.average.name], ascending=False)
-
-    df.reset_index(inplace=True)
-
-    _cols = frozenset(cols).intersection(frozenset(df.columns.to_list()))
-    df = df[_cols].round(decimals=2)
-
-    # filter out if any of the benchmarks have not been produced
-    df = df[has_no_nan_values(df, _benchmark_cols)]
-    return df
-
-
-def get_evaluation_queue_df(save_path: str, cols: list) -> list[pd.DataFrame]:
-    """Creates the different dataframes for the evaluation queues requests"""
-    # entries = [entry for entry in os.listdir(save_path) if not entry.startswith(".")]
-    # all_evals = []
-    #
-    # for entry in entries:
-    #     if ".json" in entry:
-    #         file_path = os.path.join(save_path, entry)
-    #         with open(file_path) as fp:
-    #             data = json.load(fp)
-    #
-    #         data[EvalQueueColumn.model.name] = make_clickable_model(data["model"])
-    #         data[EvalQueueColumn.revision.name] = data.get("revision", "main")
-    #
-    #         all_evals.append(data)
-    #     elif ".md" not in entry:
-    #         # this is a folder
-    #         sub_entries = [e for e in os.listdir(f"{save_path}/{entry}") if not e.startswith(".")]
-    #         for sub_entry in sub_entries:
-    #             file_path = os.path.join(save_path, entry, sub_entry)
-    #             with open(file_path) as fp:
-    #                 data = json.load(fp)
-    #
-    #             data[EvalQueueColumn.model.name] = make_clickable_model(data["model"])
-    #             data[EvalQueueColumn.revision.name] = data.get("revision", "main")
-    #             all_evals.append(data)
-    #
-    # pending_list = [e for e in all_evals if e["status"] in ["PENDING", "RERUN"]]
-    # running_list = [e for e in all_evals if e["status"] == "RUNNING"]
-    # finished_list = [e for e in all_evals if e["status"].startswith("FINISHED") or e["status"] == "PENDING_NEW_EVAL"]
-    # df_pending = pd.DataFrame.from_records(pending_list, columns=cols)
-    # df_running = pd.DataFrame.from_records(running_list, columns=cols)
-    # df_finished = pd.DataFrame.from_records(finished_list, columns=cols)
-    cols = ["Retrieval Model", "Submitted Time", "Status"]
-    df_finished = pd.DataFrame(
-        {
-            "Retrieval Model": ["bge-m3", "jina-embeddings-v2"],
-            "Submitted Time": ["2024-05-01 12:34:20", "2024-05-02 12:34:20"],
-            "Status": ["FINISHED", "FINISHED"]
-        }
-    )
-    df_running = pd.DataFrame(
-        {
-            "Retrieval Model": ["bge-m3", "jina-embeddings-v2"],
-            "Submitted Time": ["2024-05-01 12:34:20", "2024-05-02 12:34:20"],
-            "Status": ["RUNNING", "RUNNING"]
-        }
-    )
-    df_pending = pd.DataFrame(
-        {
-            "Retrieval Model": ["bge-m3", "jina-embeddings-v2"],
-            "Submitted Time": ["2024-05-01 12:34:20", "2024-05-02 12:34:20"],
-            "Status": ["PENDING", "PENDING"]
-        }
-    )
-    return df_finished, df_running, df_pending

tests/src/leaderboard/test_read_evals.py

@@ -1,6 +1,6 @@
 from pathlib import Path
 
-from src.leaderboard.read_evals import FullEvalResult, get_raw_eval_results
+from src.leaderboard.read_evals import FullEvalResult, get_raw_eval_results, get_leaderboard_df
 
 cur_fp = Path(__file__)
 
@@ -8,7 +8,11 @@ cur_fp = Path(__file__)
 def test_init_from_json_file():
     json_fp = cur_fp.parents[2] / "toydata" / "test_data.json"
     full_eval_result = FullEvalResult.init_from_json_file(json_fp)
-    assert len(full_eval_result.results) == 6
+    num_different_task_domain_lang_metric_dataset_combination = 6
+    assert len(full_eval_result.results) == \
+           num_different_task_domain_lang_metric_dataset_combination
+    assert full_eval_result.retrieval_model == "bge-m3"
+    assert full_eval_result.reranking_model == "bge-reranker-v2-m3"
 
 
 def test_to_dict():
@@ -32,3 +36,32 @@ def test_get_raw_eval_results():
     assert len(results[0].results) == 6
     assert results[1].eval_name == "bge-m3_bge-reranker-v2-m3"
     assert len(results[1].results) == 6
+
+def test_get_leaderboard_df():
+    results_path = cur_fp.parents[2] / "toydata" / "test_results"
+    raw_data = get_raw_eval_results(results_path)
+    df = get_leaderboard_df(raw_data, 'qa', 'ndcg_at_1')
+    assert df.shape[0] == 2
+    # the results contain only one embedding model
+    for i in range(2):
+        assert df["Retrieval Model"][i] == "bge-m3"
+    # the results contain only two reranking model
+    assert df["Reranking Model"][0] == "bge-reranker-v2-m3"
+    assert df["Reranking Model"][1] == "NoReranker"
+    assert df["Average ⬆️"][0] > df["Average ⬆️"][1]
+    assert not df[['Average ⬆️', 'wiki_en', 'wiki_zh',]].isnull().values.any()
+
+
+def test_get_leaderboard_df_long_doc():
+    results_path = cur_fp.parents[2] / "toydata" / "test_results"
+    raw_data = get_raw_eval_results(results_path)
+    df = get_leaderboard_df(raw_data, 'long_doc', 'ndcg_at_1')
+    assert df.shape[0] == 2
+    # the results contain only one embedding model
+    for i in range(2):
+        assert df["Retrieval Model"][i] == "bge-m3"
+    # the results contains only two reranking model
+    assert df["Reranking Model"][0] == "bge-reranker-v2-m3"
+    assert df["Reranking Model"][1] == "NoReranker"
+    assert df["Average ⬆️"][0] > df["Average ⬆️"][1]
+    assert not df[['Average ⬆️', 'law_en_lex_files_500k_600k',]].isnull().values.any()

tests/src/test_populate.py

@@ -1,41 +0,0 @@
-from src.populate import get_leaderboard_df
-from src.leaderboard.read_evals import get_raw_eval_results
-from pathlib import Path
-
-cur_fp = Path(__file__)
-
-
-def test_get_leaderboard_df():
-    requests_path = cur_fp.parents[1] / "toydata" / "test_requests"
-    results_path = cur_fp.parents[1] / "toydata" / "test_results"
-    cols = ['Retrieval Model', 'Reranking Model', 'Average ⬆️', 'wiki_en', 'wiki_zh',]
-    benchmark_cols = ['wiki_en', 'wiki_zh',]
-    raw_data = get_raw_eval_results(results_path, requests_path)
-    df = get_leaderboard_df(raw_data, cols, benchmark_cols, 'qa', 'ndcg_at_1')
-    assert df.shape[0] == 2
-    # the results contain only one embedding model
-    for i in range(2):
-        assert df["Retrieval Model"][i] == "bge-m3"
-    # the results contains only two reranking model
-    assert df["Reranking Model"][0] == "bge-reranker-v2-m3"
-    assert df["Reranking Model"][1] == "NoReranker"
-    assert df["Average ⬆️"][0] > df["Average ⬆️"][1]
-    assert not df[['Average ⬆️', 'wiki_en', 'wiki_zh',]].isnull().values.any()
-
-
-def test_get_leaderboard_df_long_doc():
-    requests_path = cur_fp.parents[1] / "toydata" / "test_requests"
-    results_path = cur_fp.parents[1] / "toydata" / "test_results"
-    cols = ['Retrieval Model', 'Reranking Model', 'Average ⬆️', 'law_en_lex_files_500k_600k',]
-    benchmark_cols = ['law_en_lex_files_500k_600k',]
-    raw_data = get_raw_eval_results(results_path, requests_path)
-    df = get_leaderboard_df(raw_data, cols, benchmark_cols, 'long_doc', 'ndcg_at_1')
-    assert df.shape[0] == 2
-    # the results contain only one embedding model
-    for i in range(2):
-        assert df["Retrieval Model"][i] == "bge-m3"
-    # the results contains only two reranking model
-    assert df["Reranking Model"][0] == "bge-reranker-v2-m3"
-    assert df["Reranking Model"][1] == "NoReranker"
-    assert df["Average ⬆️"][0] > df["Average ⬆️"][1]
-    assert not df[['Average ⬆️', 'law_en_lex_files_500k_600k',]].isnull().values.any()

tests/toydata/test_results/bge-m3/NoReranker/results_2023-11-21T18-10-08.json

@@ -11,7 +11,7 @@
         "domain": "law",
         "lang": "en",
         "dataset": "lex_files_500K-600K",
-        "value": 0.75723
+        "value": 0.45723
       }
     ]
   },
@@ -27,7 +27,7 @@
         "domain": "law",
         "lang": "en",
         "dataset": "lex_files_500K-600K",
-        "value": 0.69909
+        "value": 0.49909
       }
     ]
   },
@@ -43,7 +43,7 @@
         "domain": "wiki",
         "lang": "en",
         "dataset": "unknown",
-        "value": 0.69083
+        "value": 0.49083
       }
     ]
   },
@@ -59,7 +59,7 @@
         "domain": "wiki",
         "lang": "en",
         "dataset": "unknown",
-        "value": 0.73359
+        "value": 0.43359
       }
     ]
   },

utils.py

@@ -7,9 +7,8 @@ from huggingface_hub import HfApi
 
 from src.display.utils import AutoEvalColumnQA, AutoEvalColumnLongDoc, COLS_QA, COLS_LONG_DOC, QA_BENCHMARK_COLS, LONG_DOC_BENCHMARK_COLS
 from src.benchmarks import BENCHMARK_COLS_QA, BENCHMARK_COLS_LONG_DOC, BenchmarksQA, BenchmarksLongDoc
-from src.leaderboard.read_evals import FullEvalResult
+from src.leaderboard.read_evals import FullEvalResult, get_leaderboard_df
 from typing import List
-from src.populate import get_leaderboard_df
 
 
 def filter_models(df: pd.DataFrame, reranking_query: list) -> pd.DataFrame: