src/leaderboard/read_evals.py

import glob
import json
import math
import os
import re
from dataclasses import dataclass

import dateutil
import numpy as np

from src.about import all_tasks, g_tasks, mc_tasks, rag_tasks
from src.display.formatting import make_clickable_model
from src.display.utils import AutoEvalColumn, ModelType, Tasks, Precision, WeightType, NShotType
from src.submission.check_validity import is_model_on_hub

NUM_FEWSHOT = 0


@dataclass
class EvalResult:
    eval_name: str  # org_model_precision (uid)
    full_model: str  # org/model (path on hub)
    org: str
    model: str
    revision: str  # commit hash, "" if main
    results: dict
    precision: Precision = Precision.Unknown
    model_type: ModelType = ModelType.Unknown  # Pretrained, fine tuned, ...
    weight_type: WeightType = WeightType.Original  # Original or Adapter
    architecture: str = "Unknown"
    license: str = "?"
    lang: str = "?"
    likes: int = 0
    num_params: int = 0
    date: str = ""  # submission date of request file
    still_on_hub: bool = False
    n_shot: NShotType = NShotType.n0
    org_and_model: str = ""
    start_date: float = 0

    @classmethod
    def init_from_json_file(self, json_filepath, n_shot_num):
        """Inits the result from the specific model result file"""
        with open(json_filepath) as fp:
            data = json.load(fp)

        config = data.get("config")
        n_shot = data.get("n-shot")
        start_date = data.get("date", 0)
        chat_template = data.get("chat_template", None)
        fewshot_as_multiturn = data.get("fewshot_as_multiturn", False)

        # Precision
        precision = Precision.from_str(config.get("model_dtype"))

        # Get model and org
        org_and_model = config.get("model_name", config.get("model_args", None))
        orig_org_and_model = org_and_model
        SPICHLERZ_ORG = "speakleash/"

        if re.match(r"^pretrained=(.*/(plgkwrobel|plggspkl)/)(models/)?", org_and_model):
            org_and_model = re.sub(r"^pretrained=(.*/(plgkwrobel|plggspkl)/)(models/)?", SPICHLERZ_ORG, org_and_model)

        org_and_model = org_and_model.replace(",dtype=bfloat16", "")
        org_and_model = org_and_model.replace(",dtype=float16", "")

        org_and_model = org_and_model.replace("models/hf_v7_e1", "APT3-1B-Instruct-e1")
        org_and_model = org_and_model.replace("models/hf_v7_e2", "APT3-1B-Instruct-e2")

        org_and_model = re.sub(r"^pretrained=", "", org_and_model)
        org_and_model = re.sub(r"^model=", "", org_and_model)
        org_and_model = org_and_model.replace(",trust_remote_code=True", "")
        org_and_model = org_and_model.replace(",parallelize=True", "")
        org_and_model = org_and_model.replace(",tokenizer_backend=huggingface", "")
        org_and_model = re.sub(",base_url=[^,]+", ",API", org_and_model)

        org_and_model = re.sub(",prefix_token_id=\d+", "", org_and_model)
        org_and_model = re.sub("/$", "", org_and_model)

        model_mapping={
            'speakleash/mistral_7B-v2/spkl-only-e1_333887a5':'speakleash/Bielik-7B-v0.1',
            'speakleash/mistral_7B-v2/spkl-only_sft_v2/e1_base/spkl-only_v10wa_7e6-e2_bbc67e89':'speakleash/Bielik-7B-Instruct-v0.1',
            'meta-llama/Meta-Llama-3.1-405B-Instruct-FP8,API': 'meta-llama/Meta-Llama-3.1-405B-Instruct-FP8,API'
        }
        #map org_and_model using model_mapping
        if org_and_model in model_mapping:
            org_and_model=model_mapping[org_and_model]
        # if org_and_model=='speakleash/mistral_7B-v2/spkl-only-e1_333887a5':
        #     org_and_model='speakleash/Bielik-7B-v0.1'
        # elif org_and_model=='speakleash/mistral_7B-v2/spkl-only_sft_v2/e1_base/spkl-only_v10wa_7e6-e2_bbc67e89':
        #     org_and_model='speakleash/Bielik-7B-Instruct-v0.1'


        if chat_template:
            org_and_model += ",chat"
            if fewshot_as_multiturn:
                org_and_model += ",multiturn"

        org_and_model = org_and_model.split("/", 1)

        if len(org_and_model) == 1:
            org = None
            model = org_and_model[0]
            result_key = f"{model}" # _{precision.value.name}
        else:
            org = org_and_model[0]
            model = org_and_model[1]
            result_key = f"{org}_{model}" # _{precision.value.name}


        # if chat_template:
        #     result_key = f"{result_key}_chat"
            # model = f"{model},chat"
            # org_and_model= f"{org_and_model[1]},chat"

        full_model = "/".join(org_and_model)

        still_on_hub, err, model_config = is_model_on_hub(
            full_model.split(',')[0], config.get("model_sha", "main"), trust_remote_code=True, test_tokenizer=False
        )

        if err:
            print(full_model, err)

        architecture = "?"
        if model_config is not None:
            architectures = getattr(model_config, "architectures", None)
            if architectures:
                architecture = ";".join(architectures)

        # Extract results available in this file (some results are split in several files)
        results = {}
        for task in Tasks:
            task = task.value

            task_n_shot_num = n_shot_num
            if 'perplexity' in task.metric or task.benchmark=='polish_eq_bench': # perplexity is the same for 0-shot and 5-shot and is calculated only with 0-shot
                task_n_shot_num = 0

            # We average all scores of a given metric (not all metrics are present in all files)
            accs = np.array([v.get(task.metric, None) for k, v in data["results"].items() if
                             task.benchmark == k and n_shot.get(k, -1) == task_n_shot_num])
            if accs.size == 0 or any([acc is None for acc in accs]):
                continue

            if 'perplexity' in task.metric or 'eqbench' in task.metric:
                mean_acc = np.mean(accs)
            else:
                mean_acc = np.mean(accs) * 100.0
            results[task.benchmark] = (mean_acc, start_date)
            # results[task.benchmark] = mean_acc

        return self(
            eval_name=result_key,
            full_model=full_model,
            org=org,
            model=model,
            results=results,
            precision=precision,
            revision=config.get("model_sha", ""),
            still_on_hub=still_on_hub,
            architecture=architecture,
            n_shot=NShotType.from_str(n_shot_num),
            org_and_model=orig_org_and_model,
            start_date=start_date
        )

    def update_with_metadata(self, metadata):
        # print('UPDATE', self.full_model, self.model, self.eval_name)
        try:
            k = self.full_model.replace(',chat', '').replace(',multiturn', '')
            meta = metadata[k]
            self.model_type = ModelType.from_str(meta.get("type", "?"))
            self.num_params = meta.get("params", 0)
            self.license = meta.get("license", "?")
            self.lang = meta.get("lang", "?")
            # TODO desc name
        except KeyError:
            print(f"Could not find metadata for {self.full_model}")

    def update_with_request_file(self, requests_path):
        """Finds the relevant request file for the current model and updates info with it"""
        return
        request_file = get_request_file_for_model(requests_path, self.full_model, self.precision.value.name)

        try:
            with open(request_file, "r") as f:
                request = json.load(f)
            self.model_type = ModelType.from_str(request.get("model_type", ""))
            self.weight_type = WeightType[request.get("weight_type", "Original")]
            self.license = request.get("license", "?")
            self.likes = request.get("likes", 0)
            self.num_params = request.get("params", 0)
            self.date = request.get("submitted_time", "")
        except Exception:
            print(f"Could not find request file for {self.org}/{self.model} with precision {self.precision.value.name}")

    def to_dict(self):
        """Converts the Eval Result to a dict compatible with our dataframe display"""
        # g_tasks = [task.value.benchmark for task in Tasks if task.value.type == "generate_until"]
        # mc_tasks = [task.value.benchmark for task in Tasks if task.value.type == "multiple_choice"]
        # rag_tasks = ['polish_polqa_reranking_multiple_choice', 'polish_polqa_open_book', 'polish_poquad_open_book']
        # all_tasks = g_tasks + mc_tasks
        all_tasks_wo_polqa = [task for task in all_tasks if 'polqa' not in task]

        baselines = {task.value.benchmark: task.value.baseline*100 for task in Tasks}

        average_old = sum([v for task, v in self.results.items() if v is not None and task in all_tasks_wo_polqa]) / len(all_tasks_wo_polqa)

        average = sum([(self.results.get(task,0) - baselines.get(task, 0)) / (100 - baselines.get(task, 0)) * 100 for task in all_tasks]) / len(all_tasks)
        average_g = sum([(self.results.get(task,0) - baselines.get(task, 0)) / (100 - baselines.get(task, 0)) * 100 for task in g_tasks]) / len(g_tasks)
        average_mc = sum([(self.results.get(task,0) - baselines.get(task, 0)) / (100 - baselines.get(task, 0)) * 100 for task in mc_tasks]) / len(mc_tasks)
        average_rag = sum([(self.results.get(task,0) - baselines.get(task, 0)) / (100 - baselines.get(task, 0)) * 100 for task in rag_tasks]) / len(rag_tasks)

        data_dict = {}
        # data_dict = {
        #     "eval_name": self.eval_name,  # not a column, just a save name,
        #     AutoEvalColumn.precision.name: self.precision.value.name,
        #     AutoEvalColumn.model_type.name: self.model_type.value.name,
        #     AutoEvalColumn.model_type_symbol.name: self.model_type.value.symbol,
        #     AutoEvalColumn.weight_type.name: self.weight_type.value.name,
        #     AutoEvalColumn.architecture.name: self.architecture,
        #     AutoEvalColumn.model.name: make_clickable_model(self.full_model),
        #     AutoEvalColumn.dummy.name: self.full_model,
        #     AutoEvalColumn.revision.name: self.revision,
        #     AutoEvalColumn.average.name: average,
        #     AutoEvalColumn.license.name: self.license,
        #     AutoEvalColumn.likes.name: self.likes,
        #     AutoEvalColumn.params.name: self.num_params,
        #     AutoEvalColumn.still_on_hub.name: self.still_on_hub,
        # }
        try:
            data_dict["eval_name"] = self.eval_name
        except KeyError:
            print(f"Could not find eval name")

        try:
            data_dict[AutoEvalColumn.precision.name] = self.precision.value.name
        except KeyError:
            print(f"Could not find precision")
        except AttributeError:
            print(f"AttributeError precision")

        try:
            data_dict[AutoEvalColumn.model_type.name] = self.model_type.value.name
        except KeyError:
            print(f"Could not find model type")

        try:
            data_dict[AutoEvalColumn.model_type_symbol.name] = self.model_type.value.symbol
        except KeyError:
            print(f"Could not find model type symbol")
        except AttributeError:
            print(f"AttributeError model_type")

        try:
            data_dict[AutoEvalColumn.weight_type.name] = self.weight_type.value.name
        except KeyError:
            print(f"Could not find weight type")

        try:
            data_dict[AutoEvalColumn.architecture.name] = self.architecture
        except KeyError:
            print(f"Could not find architecture")
        except AttributeError:
            print(f"AttributeError architecture")

        try:
            data_dict[AutoEvalColumn.model.name] = make_clickable_model(
                self.full_model, self.model) if self.still_on_hub else self.model  #TODO or full_model
        except KeyError:
            print(f"Could not find model")

        try:
            data_dict[AutoEvalColumn.dummy.name] = self.full_model
        except KeyError:
            print(f"Could not find dummy")

        try:
            data_dict[AutoEvalColumn.revision.name] = self.revision
        except KeyError:
            print(f"Could not find revision")
        except AttributeError:
            print(f"AttributeError revision")

        try:
            data_dict[AutoEvalColumn.average_old.name] = average_old
        except KeyError:
            print(f"Could not find average_old")

        try:
            data_dict[AutoEvalColumn.average.name] = average
        except KeyError:
            print(f"Could not find average")

        try:
            data_dict[AutoEvalColumn.average_g.name] = average_g
        except KeyError:
            print(f"Could not find average_g")

        try:
            data_dict[AutoEvalColumn.average_mc.name] = average_mc
        except KeyError:
            print(f"Could not find average_mc")

        try:
            data_dict[AutoEvalColumn.average_rag.name] = average_rag
        except KeyError:
            print(f"Could not find average_rag")

        try:
            data_dict[AutoEvalColumn.license.name] = self.license
        except KeyError:
            print(f"Could not find license")
        except AttributeError:
            print(f"AttributeError license")

        try:
            data_dict[AutoEvalColumn.lang.name] = self.lang
        except KeyError:
            print(f"Could not find lang")
        except AttributeError:
            print(f"AttributeError lang")

        try:
            data_dict[AutoEvalColumn.likes.name] = self.likes
        except KeyError:
            print(f"Could not find likes")
        except AttributeError:
            print(f"AttributeError likes")

        try:
            data_dict[AutoEvalColumn.params.name] = self.num_params
        except KeyError:
            print(f"Could not find params")
        except AttributeError:
            print(f"AttributeError params")

        try:
            data_dict[AutoEvalColumn.still_on_hub.name] = self.still_on_hub
        except KeyError:
            print(f"Could not find still on hub")
        except AttributeError:
            print(f"AttributeError stillonhub")

        try:
            data_dict[AutoEvalColumn.n_shot.name] = self.n_shot.value.name
        except KeyError:
            print(f"Could not find still on hub")

        for task in Tasks:
            try:
                data_dict[task.value.col_name] = self.results[task.value.benchmark]
            except KeyError:
                print(f"Could not find {task.value.col_name}")
                data_dict[task.value.col_name] = None

        data_dict[AutoEvalColumn.rank.name] = 0

        return data_dict


def get_request_file_for_model(requests_path, model_name, precision):
    """Selects the correct request file for a given model. Only keeps runs tagged as FINISHED"""
    request_files = os.path.join(
        requests_path,
        f"{model_name}_eval_request_*.json",
    )
    request_files = glob.glob(request_files)

    # Select correct request file (precision)
    request_file = ""
    request_files = sorted(request_files, reverse=True)
    for tmp_request_file in request_files:
        with open(tmp_request_file, "r") as f:
            req_content = json.load(f)
            if (
                    req_content["status"] in ["FINISHED"]
                    and req_content["precision"] == precision.split(".")[-1]
            ):
                request_file = tmp_request_file
    return request_file


def get_raw_eval_results(results_path: str, requests_path: str, metadata) -> list[EvalResult]:
    """From the path of the results folder root, extract all needed info for results"""
    model_result_filepaths = []

    for root, _, files in os.walk(results_path):
        # We should only have json files in model results
        if len(files) == 0 or any([not f.endswith(".json") for f in files]):
            continue

        # Sort the files by date
        try:
            files.sort(key=lambda x: x.removesuffix(".json").removeprefix("results_")[:-7])
        except dateutil.parser._parser.ParserError:
            files = [files[-1]]

        for file in files:
            model_result_filepaths.append(os.path.join(root, file))

    # print('PATHS:', model_result_filepaths)

    eval_results = {}
    for n_shot in [0, 5]:
        for model_result_filepath in model_result_filepaths:
            # Creation of result
            eval_result = EvalResult.init_from_json_file(model_result_filepath, n_shot_num=n_shot)
            eval_result.update_with_request_file(requests_path)
            # update with metadata
            eval_result.update_with_metadata(metadata)

            # Store results of same eval together
            eval_name = f"{eval_result.eval_name}_{n_shot}-shot"
            if eval_name in eval_results.keys():

                for k, (v, start_date) in eval_result.results.items():
                    if v is not None:
                        if k in eval_results[eval_name].results:
                            if start_date > eval_results[eval_name].results[k][1]:
                                print(
                                    f"Overwriting {eval_name}.results {k} {eval_results[eval_name].results[k]} with {v}: {model_result_filepath} {n_shot} {eval_result.start_date}  {eval_results[eval_name].start_date}")
                                eval_results[eval_name].results[k] = (v, start_date)
                            else:
                                print(
                                    f"Skipping {eval_name} {eval_result.start_date} {eval_results[eval_name].start_date}: {model_result_filepath} {n_shot}")
                        else:
                            eval_results[eval_name].results[k] = (v, start_date)
                # eval_results[eval_name].results.update({k: v for k, v in eval_result.results.items() if v is not None})
                # TODO: log updated

            else:
                eval_results[eval_name] = eval_result

    for k,v in eval_results.items():
        v.results = {k: v for k, (v, start_date) in v.results.items()}



    all_models = []
    missing_results_for_task = {}
    missing_metadata = []
    for_run=[]
    for v in eval_results.values():
        r = v.to_dict()
        in_progress=False
        for task in Tasks:
            if r[task.value.col_name] is None:
                task_name = f"{r['n_shot']}|{task.value.benchmark}"
                if task_name in missing_results_for_task:
                    missing_results_for_task[task_name].append(f"{v.full_model}|{v.org_and_model}")
                    if v.still_on_hub and task.value.benchmark in all_tasks:
                        for_run.append([r["n_shot"], task.value.benchmark, v.full_model])
                        in_progress=True
                        # print(f'sbatch start.sh "bash eval_model_task_bs1.sh {r["n_shot"]} {task.value.benchmark} {v.full_model}"')
                else:
                    missing_results_for_task[task_name] = [f"{v.full_model}|{v.org_and_model}"]
                    if v.still_on_hub and task.value.benchmark in all_tasks:
                        for_run.append([r["n_shot"], task.value.benchmark, v.full_model])
                        in_progress=True
                        # print(f'sbatch start.sh "bash eval_model_task_bs1.sh {r["n_shot"]} {task.value.benchmark} {v.full_model}"')
        if in_progress:
            v.model = '🚧' + v.model

        if r[AutoEvalColumn.lang.name] is None or r[AutoEvalColumn.lang.name] == "?":
            missing_metadata.append(f"{v.full_model}")
        all_models.append((v.full_model, v.num_params, v.still_on_hub))

    results = []
    for v in eval_results.values():
        try:
            print(v)
            v.to_dict()  # we test if the dict version is complete
            # if v.results:
            results.append(v)
        except KeyError:  # not all eval values present
            print(f"not all eval values present {v.eval_name} {v.full_model}")
            continue

    print(f"Missing sbatch results:")
    for r in for_run:
        if r[0]==5 and r[1] in ['polish_eq_bench']: continue
        fm=r[2]
        script='eval_model_task_bs1.sh'
        if ',chat' in fm:
            script='eval_model_task_bs1_chat.sh'
            fm=fm.replace(',chat','')
        if ',multiturn' in fm:
            script='eval_model_task_bs1_chat_few.sh'
            fm=fm.replace(',multiturn','')

        print(f'sbatch start.sh "bash {script} {r[0]} {r[1]} {fm}"')


    # print('missing_results_for_task', missing_results_for_task)
    for task, models in missing_results_for_task.items():
        print(f"Missing results for {task} for {len(models)} models")
        # print(" ".join(models))
        for model in models:
            print(f'"{model}"')
        print()

    print(f"Missing metadata for {len(missing_metadata)} models")
    for model in missing_metadata:
        print(model)
    print()

    print(f"All models:")
    for model in all_models:
        print(model)
    print()

    return results