negbleurt / negbleurt.py
MiriUll's picture
Update negbleurt.py
81dbb49
raw history blame
No virus
3.16 kB
from transformers import AutoTokenizer, AutoModelForSequenceClassification
import datasets
import evaluate
_CITATION = """\
tba
"""
_DESCRIPTION = """\
Negation-aware version of BLEURT metric.
BLEURT a learnt evaluation metric for Natural Language Generation. It is built using multiple phases of transfer learning starting from a pretrained BERT model (Devlin et al. 2018) and then employing another pre-training phrase using synthetic data. Finally it is trained on WMT human annotations and the CANNOT negation awareness dataset.
"""
_KWARGS_DESCRIPTION = """
Calculates the NegBLEURT scores between references and predictions
Args:
predictions: list of predictions to score. Each prediction should be a string.
references: single reference or list of references for each prediction. If only one reference is given, all predictions will be scored against the same reference
batch_size: batch_size for model inference. Default is 16
Returns:
negBLEURT: List of NegBLEURT scores for all predictions
Examples:
>>> negBLEURT = evaluate.load('MiriUll/negbleurt')
>>> predictions = ["Ray Charles is a legend.", "Ray Charles isn’t legendary."]
>>> reference = "Ray Charles is legendary."
>>> results = rouge.compute(predictions=predictions, references=reference)
>>> print(results)
{'negBLERUT': [0.8409, 0.2601]}
"""
@evaluate.utils.file_utils.add_start_docstrings(_DESCRIPTION, _KWARGS_DESCRIPTION)
class NegBLEURT(evaluate.Metric):
def _info(self):
return evaluate.MetricInfo(
description=_DESCRIPTION,
citation=_CITATION,
inputs_description=_KWARGS_DESCRIPTION,
features=[
datasets.Features(
{
"predictions": datasets.Value("string", id="sequence"),
"references": datasets.Sequence(datasets.Value("string", id="sequence")),
}
),
datasets.Features(
{
"predictions": datasets.Value("string", id="sequence"),
"references": datasets.Value("string", id="sequence"),
}
),
],
codebase_urls=["https://github.com/MiriUll/negation_aware_evaluation"]
)
def _download_and_prepare(self, dl_manager):
model_name = "tum-nlp/NegBLEURT"
self.tokenizer = AutoTokenizer.from_pretrained(model_name)
self.model = AutoModelForSequenceClassification.from_pretrained(model_name)
def _compute(
self, predictions, references, batch_size=16
):
single_ref = isinstance(references, str)
if single_ref:
references = [references] * len(predictions)
scores_negbleurt = []
for i in tqdm(range(0, len(references), batch_size)):
tokenized = self.tokenizer(references[i:i+batch_size], candidates[i:i+batch_size], return_tensors='pt', padding=True, max_length=512, truncation=True)
scores_negbleurt += self.model(**tokenized).logits.flatten().tolist()
return {'negBLEURT': scores_negbleurt}