scripts/repro_matrix/run_all_miracl.py

#
# Pyserini: Reproducible IR research with sparse and dense representations
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#

import argparse
import math
import os
import subprocess
import time
from collections import defaultdict
from string import Template

import yaml

from scripts.repro_matrix.defs_miracl import models, languages, trec_eval_metric_definitions
from scripts.repro_matrix.utils import run_eval_and_return_metric, ok_str, okish_str, fail_str


def print_results(metric, split):
    print(f'Metric = {metric}, Split = {split}')
    print(' ' * 35, end='')
    for lang in languages:
        print(f'{lang[0]:3}    ', end='')
    print('')
    for model in models:
        print(f'{model:33}', end='')
        for lang in languages:
            key = f'{model}.{lang[0]}'
            print(f'{table[key][split][metric]:7.3f}', end='')
        print('')
    print('')


def extract_topic_fn_from_cmd(cmd):
    cmd = cmd.split()
    topic_idx = cmd.index('--topics')
    return cmd[topic_idx + 1]


if __name__ == '__main__':
    parser = argparse.ArgumentParser(description='Generate regression matrix for MIRACL.')
    parser.add_argument('--skip-eval', action='store_true', default=False, help='Skip running trec_eval.')
    args = parser.parse_args()

    start = time.time()

    table = defaultdict(lambda: defaultdict(lambda: defaultdict(lambda: 0.0)))

    with open('pyserini/resources/miracl.yaml') as f:
        yaml_data = yaml.safe_load(f)
        for condition in yaml_data['conditions']:
            name = condition['name']
            eval_key = condition['eval_key']
            cmd_template = condition['command']
            cmd_lst = cmd_template.split()

            print(f'condition {name}:')
            lang = name.split('.')[-1]
            is_hybrid_run = 'hybrid' in name

            for splits in condition['splits']:
                split = splits['split']
                if is_hybrid_run:
                    hits = int(cmd_lst[cmd_lst.index('--k') + 1])
                else:
                    hits = int(cmd_lst[cmd_lst.index('--hits') + 1])

                print(f'  - split: {split}')

                runfile = f'runs/run.miracl.{name}.{split}.top{hits}.txt'
                if is_hybrid_run:
                    bm25_output = f'runs/run.miracl.bm25.{lang}.{split}.top{hits}.txt'
                    mdpr_output = f'runs/run.miracl.mdpr-tied-pft-msmarco.{lang}.{split}.top{hits}.txt'
                    if not os.path.exists(bm25_output):
                        print(f'Missing BM25 file: {bm25_output}')
                        continue
                    if not os.path.exists(mdpr_output):
                        print(f'Missing mDPR file: {mdpr_output}')
                        continue
                    cmd = Template(cmd_template).substitute(split=split, output=runfile, bm25_output=bm25_output, mdpr_output=mdpr_output)
                else:
                    cmd = Template(cmd_template).substitute(split=split, output=runfile)

                # In the yaml file, the topics are written as something like '--topics miracl-v1.0-ar-${split}'
                # This works for the dev split because the topics are directly included in Anserini/Pyserini.
                # For this training split, we have to map the symbol into a file in tools/topics-and-qrels/
                # Here, we assume that the developer has cloned the miracl repo and placed the topics there.
                if split == 'train':
                    cmd = cmd.replace(f'--topics miracl-v1.0-{lang}-{split}',
                                      f'--topics tools/topics-and-qrels/topics.miracl-v1.0-{lang}-{split}.tsv')

                if not os.path.exists(runfile):
                    print(f'    Running: {cmd}')
                    rtn = subprocess.run(cmd.split(), capture_output=True)
                    stderr = rtn.stderr.decode()
                    if '--topics' in cmd:
                        topic_fn = extract_topic_fn_from_cmd(cmd)
                        if f'ValueError: Topic {topic_fn} Not Found' in stderr:
                            print(f'Skipping {topic_fn}: file not found.')
                            continue

                for expected in splits['scores']:
                    for metric in expected:
                        if not args.skip_eval:
                            # We have the translate the training qrels into a file located in tools/topics-and-qrels/
                            # because they are not included with Anserini/Pyserini by default.
                            # Here, we assume that the developer has cloned the miracl repo and placed the qrels there.
                            if split == 'train':
                                qrels = f'tools/topics-and-qrels/qrels.{eval_key}-train.tsv'
                            else:
                                qrels = f'{eval_key}-{split}'
                            score = float(run_eval_and_return_metric(metric, qrels,
                                                                     trec_eval_metric_definitions[metric], runfile))
                            if math.isclose(score, float(expected[metric])):
                                result_str = ok_str
                            # Flaky tests
                            elif (name == 'mdpr-tied-pft-msmarco.hi' and split == 'train'
                                  and math.isclose(score, float(expected[metric]), abs_tol=2e-4)) or \
                                 (name == 'mdpr-tied-pft-msmarco-ft-all.ru'
                                  and split == 'dev' and metric == 'nDCG@10'
                                  and math.isclose(score, float(expected[metric]), abs_tol=2e-4)) or \
                                 (name == 'bm25-mdpr-tied-pft-msmarco-hybrid.te'
                                  and split == 'train' and metric == 'nDCG@10'
                                  and math.isclose(score, float(expected[metric]), abs_tol=2e-4)) or \
                                 (name == 'bm25-mdpr-tied-pft-msmarco-hybrid.zh'
                                  and split == 'dev' and metric == 'nDCG@10'
                                  and math.isclose(score, float(expected[metric]), abs_tol=2e-4)):
                                result_str = okish_str
                            else:
                                result_str = fail_str + f' expected {expected[metric]:.4f}'
                            print(f'      {metric:7}: {score:.4f} {result_str}')
                            table[name][split][metric] = score
                        else:
                            table[name][split][metric] = expected[metric]

            print('')

    for metric in ['nDCG@10', 'R@100']:
        for split in ['dev', 'train']:
            print_results(metric, split)

    end = time.time()
    print(f'Total elapsed time: {end - start:.0f}s')