|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
"""ANLS - Average Normalized Levenshtein Similarity""" |
|
|
|
import datasets |
|
import evaluate |
|
|
|
from compute_score import compute_score |
|
|
|
|
|
_CITATION = """\ |
|
@article{, |
|
title = {Binary codes capable of correcting deletions, insertions, and reversals}, |
|
journal = {Soviet physics doklady}, |
|
volume = {10}, |
|
number = {8}, |
|
pages = {707--710}, |
|
year = {1966}, |
|
url = {https://nymity.ch/sybilhunting/pdf/Levenshtein1966a.pdf}, |
|
author = {V. I. Levenshtein}, |
|
} |
|
""" |
|
|
|
_DESCRIPTION = """\ |
|
ANLS refer to the average normalized Levenshtein similarity. |
|
""" |
|
|
|
|
|
_KWARGS_DESCRIPTION = """ |
|
Computes Average Normalized Levenshtein Similarity (ANLS). |
|
Args: |
|
predictions: List of question-answers dictionaries with the following key-values: |
|
- 'id': id of the question-answer pair as given in the references (see below) |
|
- 'prediction_text': the text of the answer |
|
references: List of question-answers dictionaries with the following key-values: |
|
- 'id': id of the question-answer pair (see above), |
|
- 'answers': a Dict in the SQuAD dataset format |
|
{ |
|
'text': list of possible texts for the answer, as a list of strings |
|
'answer_start': list of start positions for the answer, as a list of ints |
|
} |
|
Note that answer_start values are not taken into account to compute the metric. |
|
Returns: |
|
'anls': The ANLS score of predicted tokens versus the gold answer |
|
Examples: |
|
>>> predictions = [{'prediction_text': '1976', 'id': '56e10a3be3433e1400422b22'}] |
|
>>> references = [{'answers': {'answer_start': [97], 'text': ['1976']}, 'id': '56e10a3be3433e1400422b22'}] |
|
>>> anls_metric = evaluate.load("anls") |
|
>>> results = anls_metric.compute(predictions=predictions, references=references) |
|
>>> print(results) |
|
{'anls_score': 100.0} |
|
""" |
|
|
|
|
|
@evaluate.utils.file_utils.add_start_docstrings(_DESCRIPTION, _KWARGS_DESCRIPTION) |
|
class Anls(evaluate.Metric): |
|
def _info(self): |
|
return evaluate.MetricInfo( |
|
description=_DESCRIPTION, |
|
citation=_CITATION, |
|
inputs_description=_KWARGS_DESCRIPTION, |
|
features=datasets.Features( |
|
{ |
|
"predictions": {"id": datasets.Value("string"), "prediction_text": datasets.Value("string")}, |
|
"references": { |
|
"id": datasets.Value("string"), |
|
"answers": datasets.features.Sequence( |
|
{ |
|
"text": datasets.Value("string"), |
|
"answer_start": datasets.Value("int32"), |
|
} |
|
), |
|
}, |
|
} |
|
) |
|
) |
|
|
|
def _compute(self, predictions, references): |
|
prediction_dict = {prediction["id"]: prediction["prediction_text"] for prediction in predictions} |
|
dataset = [ |
|
{ |
|
"paragraphs": [ |
|
{ |
|
"qas": [ |
|
{ |
|
"answers": [{"text": answer_text} for answer_text in ref["answers"]["text"]], |
|
"id": ref["id"], |
|
} |
|
for ref in references |
|
] |
|
} |
|
] |
|
} |
|
] |
|
score = compute_score(dataset=dataset, predictions=prediction_dict) |
|
return score |
|
|