Spaces:
Runtime error
Runtime error
| # | |
| # Pyserini: Reproducible IR research with sparse and dense representations | |
| # | |
| # Licensed under the Apache License, Version 2.0 (the "License"); | |
| # you may not use this file except in compliance with the License. | |
| # You may obtain a copy of the License at | |
| # | |
| # http://www.apache.org/licenses/LICENSE-2.0 | |
| # | |
| # Unless required by applicable law or agreed to in writing, software | |
| # distributed under the License is distributed on an "AS IS" BASIS, | |
| # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. | |
| # See the License for the specific language governing permissions and | |
| # limitations under the License. | |
| # | |
| from collections import defaultdict | |
| from string import Template | |
| import yaml | |
| from scripts.repro_matrix.defs_mrtydi import models, languages, html_display, trec_eval_metric_definitions | |
| def format_run_command(raw): | |
| return raw.replace('--lang', '\\\n --lang')\ | |
| .replace('--encoder', '\\\n --encoder')\ | |
| .replace('--topics', '\\\n --topics')\ | |
| .replace('--index', '\\\n --index')\ | |
| .replace('--output ', '\\\n --output ')\ | |
| .replace('--batch ', '\\\n --batch ') \ | |
| .replace('--threads 12', '--threads 12 \\\n ') | |
| def format_eval_command(raw): | |
| return raw.replace('-c ', '\\\n -c ')\ | |
| .replace('run.', '\\\n run.') | |
| def read_file(f): | |
| fin = open(f, 'r') | |
| text = fin.read() | |
| fin.close() | |
| return text | |
| def generate_table_rows(table_id, split, metric): | |
| row_cnt = 1 | |
| html_rows = [] | |
| for model in models: | |
| s = Template(row_template) | |
| keys = {} | |
| for lang in languages: | |
| keys[lang[0]] = f'{model}.{lang[0]}' | |
| sum = table[keys["ar"]][split][metric] + \ | |
| table[keys["bn"]][split][metric] + \ | |
| table[keys["en"]][split][metric] + \ | |
| table[keys["fi"]][split][metric] + \ | |
| table[keys["id"]][split][metric] + \ | |
| table[keys["ja"]][split][metric] + \ | |
| table[keys["ko"]][split][metric] + \ | |
| table[keys["ru"]][split][metric] + \ | |
| table[keys["sw"]][split][metric] + \ | |
| table[keys["te"]][split][metric] + \ | |
| table[keys["th"]][split][metric] | |
| avg = sum / 11 | |
| s = s.substitute(table_cnt=table_id, | |
| row_cnt=row_cnt, | |
| model=html_display[model], | |
| ar=f'{table[keys["ar"]][split][metric]:.3f}', | |
| bn=f'{table[keys["bn"]][split][metric]:.3f}', | |
| en=f'{table[keys["en"]][split][metric]:.3f}', | |
| fi=f'{table[keys["fi"]][split][metric]:.3f}', | |
| id=f'{table[keys["id"]][split][metric]:.3f}', | |
| ja=f'{table[keys["ja"]][split][metric]:.3f}', | |
| ko=f'{table[keys["ko"]][split][metric]:.3f}', | |
| ru=f'{table[keys["ru"]][split][metric]:.3f}', | |
| sw=f'{table[keys["sw"]][split][metric]:.3f}', | |
| te=f'{table[keys["te"]][split][metric]:.3f}', | |
| th=f'{table[keys["th"]][split][metric]:.3f}', | |
| avg=f'{avg:.3f}', | |
| cmd1=f'{commands[keys["ar"]]}', | |
| cmd2=f'{commands[keys["bn"]]}', | |
| cmd3=f'{commands[keys["en"]]}', | |
| cmd4=f'{commands[keys["fi"]]}', | |
| cmd5=f'{commands[keys["id"]]}', | |
| cmd6=f'{commands[keys["ja"]]}', | |
| cmd7=f'{commands[keys["ko"]]}', | |
| cmd8=f'{commands[keys["ru"]]}', | |
| cmd9=f'{commands[keys["sw"]]}', | |
| cmd10=f'{commands[keys["te"]]}', | |
| cmd11=f'{commands[keys["th"]]}', | |
| eval_cmd1=f'{eval_commands[keys["ar"]][metric]}', | |
| eval_cmd2=f'{eval_commands[keys["bn"]][metric]}', | |
| eval_cmd3=f'{eval_commands[keys["en"]][metric]}', | |
| eval_cmd4=f'{eval_commands[keys["fi"]][metric]}', | |
| eval_cmd5=f'{eval_commands[keys["id"]][metric]}', | |
| eval_cmd6=f'{eval_commands[keys["ja"]][metric]}', | |
| eval_cmd7=f'{eval_commands[keys["ko"]][metric]}', | |
| eval_cmd8=f'{eval_commands[keys["ru"]][metric]}', | |
| eval_cmd9=f'{eval_commands[keys["sw"]][metric]}', | |
| eval_cmd10=f'{eval_commands[keys["te"]][metric]}', | |
| eval_cmd11=f'{eval_commands[keys["th"]][metric]}' | |
| ) | |
| html_rows.append(s) | |
| row_cnt += 1 | |
| return html_rows | |
| if __name__ == '__main__': | |
| table = defaultdict(lambda: defaultdict(lambda: defaultdict(lambda: 0.0))) | |
| commands = defaultdict(lambda: '') | |
| eval_commands = defaultdict(lambda: defaultdict(lambda: '')) | |
| html_template = read_file('scripts/repro_matrix/mrtydi_html.template') | |
| table_template = read_file('scripts/repro_matrix/mrtydi_html_table.template') | |
| row_template = read_file('scripts/repro_matrix/mrtydi_html_table_row.template') | |
| with open('pyserini/resources/mrtydi.yaml') as f: | |
| yaml_data = yaml.safe_load(f) | |
| for condition in yaml_data['conditions']: | |
| name = condition['name'] | |
| eval_key = condition['eval_key'] | |
| cmd_template = condition['command'] | |
| for splits in condition['splits']: | |
| split = splits['split'] | |
| runfile = f'run.mrtydi.{name}.{split}.txt' | |
| cmd = Template(cmd_template).substitute(split=split, output=runfile) | |
| commands[name] = format_run_command(cmd) | |
| for expected in splits['scores']: | |
| for metric in expected: | |
| table[name][split][metric] = expected[metric] | |
| eval_cmd = f'python -m pyserini.eval.trec_eval ' + \ | |
| f'{trec_eval_metric_definitions[metric]} {eval_key}-{split} {runfile}' | |
| eval_commands[name][metric] = format_eval_command(eval_cmd) | |
| tables_html = [] | |
| # Build the table for MRR@100, test queries | |
| html_rows = generate_table_rows(1, 'test', 'MRR@100') | |
| all_rows = '\n'.join(html_rows) | |
| tables_html.append(Template(table_template).substitute(desc='MRR@100, test queries', rows=all_rows)) | |
| # Build the table for R@100, test queries | |
| html_rows = generate_table_rows(2, 'test', 'R@100') | |
| all_rows = '\n'.join(html_rows) | |
| tables_html.append(Template(table_template).substitute(desc='Recall@100, test queries', rows=all_rows)) | |
| print(Template(html_template).substitute(title='Mr.TyDi', tables=' '.join(tables_html))) | |