semf1.py

# Copyright 2020 The HuggingFace Datasets Authors and the current dataset script contributor.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
# TODO: Add test cases, Remove tokenize_sentences flag since it can be determined from the input itself.
"""Sem-F1 metric"""

import abc
import sys
from typing import List, Optional, Tuple, Union

import datasets
import evaluate
import nltk
import numpy as np
from numpy.typing import NDArray
from sentence_transformers import SentenceTransformer
from sklearn.metrics.pairwise import cosine_similarity
import torch

_CITATION = """\
@inproceedings{bansal-etal-2022-sem,
    title = "{SEM}-F1: an Automatic Way for Semantic Evaluation of Multi-Narrative Overlap Summaries at Scale",
    author = "Bansal, Naman  and
      Akter, Mousumi  and
      Karmaker Santu, Shubhra Kanti",
    editor = "Goldberg, Yoav  and
      Kozareva, Zornitsa  and
      Zhang, Yue",
    booktitle = "Proceedings of the 2022 Conference on Empirical Methods in Natural Language Processing",
    month = dec,
    year = "2022",
    address = "Abu Dhabi, United Arab Emirates",
    publisher = "Association for Computational Linguistics",
    url = "https://aclanthology.org/2022.emnlp-main.49",
    doi = "10.18653/v1/2022.emnlp-main.49",
    pages = "780--792",
    abstract = "Recent work has introduced an important yet relatively under-explored NLP task called Semantic Overlap Summarization (SOS) that entails generating a summary from multiple alternative narratives which conveys the common information provided by those narratives. Previous work also published a benchmark dataset for this task by collecting 2,925 alternative narrative pairs from the web and manually annotating 411 different reference summaries by engaging human annotators. In this paper, we exclusively focus on the automated evaluation of the SOS task using the benchmark dataset. More specifically, we first use the popular ROUGE metric from text-summarization literature and conduct a systematic study to evaluate the SOS task. Our experiments discover that ROUGE is not suitable for this novel task and therefore, we propose a new sentence-level precision-recall style automated evaluation metric, called SEM-F1 (Semantic F1). It is inspired by the benefits of the sentence-wise annotation technique using overlap labels reported by the previous work. Our experiments show that the proposed SEM-F1 metric yields a higher correlation with human judgment and higher inter-rater agreement compared to the ROUGE metric.",
}
"""

_DESCRIPTION = """\
Sem-F1 metric leverages the pre-trained contextual embeddings and evaluates the model generated 
semantic overlap summary with the reference overlap summary. It evaluates the semantic overlap summary at the 
sentence level and computes precision, recall and F1 scores.
"""

_KWARGS_DESCRIPTION = """
Sem-F1 compares the system generated overlap summary with ground truth reference overlap.

Args:
    predictions: list - List of  predictions (Details below)
    references: list - List of references (Details below)
        reference should be a string with tokens separated by spaces.
    model_type: str - Model to use. [pv1, stsb, use] 
        Options: 
            pv1 - paraphrase-distilroberta-base-v1 (Default)
            stsb - stsb-roberta-large 
            use - Universal Sentence Encoder
    tokenize_sentences: bool - Sentence tokenize the input document (prediction/reference). Default: True.
    gpu: Union[bool, int] - Whether to use GPU or CPU. 
        Options: 
            False - CPU (Default)
            True - GPU, device 0
            n: int - GPU, device n
    batch_size: int - Batch Size, Default = 32.
Returns:
    precision: Precision.
    recall: Recall.
    f1: F1 score.
    
There are 4 possible cases for inputs corresponding to predictions and references arguments 
Case 1: Multi-Ref = False, tokenize_sentences = False
    predictions: List[List[str]] - List of  predictions where each prediction is a list of sentences.
    references: List[List[str]] - List of references where each reference is a list of sentences.
Case 2: Multi-Ref = False, tokenize_sentences = True
    predictions: List[str] - List of  predictions where each prediction is a document
    references: List[str] - List of references where each reference is a document
Case 3: Multi-Ref = True, tokenize_sentences = False
    predictions: List[List[str]] - List of  predictions where each prediction is a list of sentences.
    references: List[List[List[str]]] - List of multi-references i.e. [[r11, r12, ...], [r21, r22, ...], ...] 
                                        where each rij is further a list of sentences
Case 4: Multi-Ref = True, tokenize_sentences = True
    predictions: List[str] - List of  predictions where each prediction is a document
    references: List[List[str]] - List of multi-references i.e. [[r11, r12, ...], [r21, r22, ...], ...]
                                  where each rij is a document  
    
This can be seen in the form of truth table as follows:
Case | Multi-Ref | tokenize_sentences | predictions     | references
1    | 0         | 0                  | List[List[str]] | List[List[str]]        
2    | 0         | 1                  | List[str]       | List[str]              
3    | 1         | 0                  | List[List[str]] | List[List[List[str]]]
4    | 1         | 1                  | List[str]       | List[List[str]]      

It is automatically determined whether it is Multi-Ref case Single-Ref case.
 
Examples:

    >>> import evaluate
    >>> predictions = [
        ["I go to School.", "You are stupid."],
        ["I love adventure sports."],
    ]
    >>> references = [
        ["I go to School.", "You are stupid."],
        ["I love adventure sports."],
    ]
    >>> metric = evaluate.load("semf1")
    >>> results = metric.compute(predictions=predictions, references=references)
    >>> print([round(v, 2) for v in results["f1"]])
    [0.77, 0.56]
"""


class Encoder(metaclass=abc.ABCMeta):
    @abc.abstractmethod
    def encode(self, prediction: List[str]) -> NDArray:
        pass


class USE(Encoder):
    def __init__(self):
        pass

    def encode(self, prediction: List[str]) -> NDArray:
        pass


class SBertEncoder(Encoder):
    def __init__(self, model_name: str, device: Union[str, int], batch_size: int):
        self.model = SentenceTransformer(model_name)
        self.device = device
        self.batch_size = batch_size

    def encode(self, prediction: List[str]) -> NDArray:
        """Returns sentence embeddings of dim: Batch x Dim"""
        # SBert output is always Batch x Dim
        return self.model.encode(prediction, device=self.device, batch_size=self.batch_size)


def _get_encoder(model_name: str, device: Union[str, int], batch_size: int) -> Encoder:
    if model_name == "use":
        return SBertEncoder(model_name, device)
        # return USE()  # TODO: This will change depending on PyTorch USE VS TF USE model
    else:
        return SBertEncoder(model_name, device, batch_size)


def _compute_f1(p, r, eps=sys.float_info.epsilon):
    '''
    Computes F1 value
    :param p: Precision Value
    :param r: Recall Value
    :return:
    '''
    f1 = 2 * p * r / (p + r + eps)
    return f1


def _compute_cosine_similarity(pred_embeds: NDArray, ref_embeds: NDArray) -> Tuple[float, float]:
    cosine_scores = cosine_similarity(pred_embeds, ref_embeds)
    precision_per_sentence_sim = np.max(cosine_scores, axis=-1)
    recall_per_sentence_sim = np.max(cosine_scores, axis=0)
    return np.mean(precision_per_sentence_sim).item(), np.mean(recall_per_sentence_sim).item()


@evaluate.utils.file_utils.add_start_docstrings(_DESCRIPTION, _KWARGS_DESCRIPTION)
class SemF1(evaluate.Metric):
    _MODEL_TYPE_TO_NAME = {
        "pv1": "paraphrase-distilroberta-base-v1",
        "stsb": "stsb-roberta-large",
        "use": "sentence-transformers/use-cmlm-multilingual",  # TODO: check PyTorch USE VS TF USE
    }

    def _info(self):
        return evaluate.MetricInfo(
            # This is the description that will appear on the modules page.
            module_type="metric",
            description=_DESCRIPTION,
            citation=_CITATION,
            inputs_description=_KWARGS_DESCRIPTION,
            # This defines the format of each prediction and reference
            features=[
                # Multi References: False, Tokenize_Sentences = False
                datasets.Features(
                    {
                        # predictions: List[List[str]] - List of predictions where prediction is a list of sentences
                        "predictions": datasets.Sequence(datasets.Value("string", id="sequence"), id="predictions"),
                        # references: List[List[str]] - List of references where each reference is a list of sentences
                        "references": datasets.Sequence(datasets.Value("string", id="sequence"), id="references"),
                    }
                ),
                # Multi References: False, Tokenize_Sentences = True
                datasets.Features(
                    {
                        # predictions: List[str] - List of predictions
                        "predictions": datasets.Value("string", id="sequence"),
                        # references: List[str] - List of documents
                        "references": datasets.Value("string", id="sequence"),
                    }
                ),
                # Multi References: True, Tokenize_Sentences = False
                datasets.Features(
                    {
                        # predictions: List[List[str]] - List of predictions where prediction is a list of sentences
                        "predictions": datasets.Sequence(datasets.Value("string", id="sequence"), id="predictions"),
                        # references: List[List[List[str]]] - List of multi-references.
                        #                                     So each "reference" is also a list (r1, r2, ...).
                        #                                     Further, each ri's are also list of sentences.
                        "references": datasets.Sequence(
                            datasets.Sequence(datasets.Value("string", id="sequence"), id="ref"), id="references"),
                    }
                ),
                # Multi References: True, Tokenize_Sentences = True
                datasets.Features(
                    {
                        # predictions: List[str] - List of predictions
                        "predictions": datasets.Value("string", id="sequence"),
                        # references: List[List[List[str]]] - List of multi-references.
                        #                                     So each "reference" is also a list (r1, r2, ...).
                        "references": datasets.Sequence(datasets.Value("string", id="ref"), id="references"),
                    }
                ),
            ],
            # # Homepage of the module for documentation
            # Additional links to the codebase or references
            reference_urls=["https://aclanthology.org/2022.emnlp-main.49/"]
        )

    def _get_model_name(self, model_type: Optional[str] = None) -> str:
        if model_type is None:
            model_type = "pv1"  # TODO: Change it to use

        if model_type not in self._MODEL_TYPE_TO_NAME.keys():
            raise ValueError(f"Provide a correct model_type.\n"
                             f"Options: {self._MODEL_TYPE_TO_NAME.keys()}\n"
                             f"Currently provided: {model_type}")

        return self._MODEL_TYPE_TO_NAME[model_type]

    def _download_and_prepare(self, dl_manager):
        """Optional: download external resources useful to compute the scores"""
        import nltk
        nltk.download("punkt", quiet=True)
        # if not nltk.data.find("tokenizers/punkt"):


    def _compute(
            self,
            predictions,
            references,
            model_type: Optional[str] = None,
            tokenize_sentences: bool = True,
            gpu: Union[bool, int] = False,
            batch_size: int = 32,
    ):

        # Ensure gpu index is within the range of total available gpus
        gpu_available = True if torch.cuda.is_available() else False
        if gpu_available:
            gpu_count = torch.cuda.device_count()
            if isinstance(gpu, int) and gpu >= gpu_count:
                raise ValueError(
                    f"There are {gpu_count} gpus available. Provide the correct gpu index. You provided: {gpu}"
                )

        # get the device
        if gpu is False:
            device = "cpu"
        elif gpu is True and torch.cuda.is_available():
            device = 0  # by default run on device 0
        elif isinstance(gpu, int):
            device = gpu
        else:  # This will never happen
            raise ValueError(f"gpu must be bool or int. Provided value: {gpu}")

        # TODO: Also have a check on references to ensure they are also in correct format
        # Ensure prediction documents are not already tokenized if tokenize_sentences is True
        if not isinstance(predictions[0], str) and tokenize_sentences:
            raise ValueError(f"Each prediction/reference should be a document i.e. when tokenize_sentences is True. "
                             f"Currently, each prediction is of type {type(predictions[0])} ")

        # Check single reference or multi-reference case
        multi_references = False
        if tokenize_sentences:
            # references: List[List[reference]]
            if isinstance(references[0], list) and isinstance(references[0][0], str):
                multi_references = True
        else:
            # references: List[List[List[sentence]]]
            if (
                    isinstance(references[0], list) and
                    isinstance(references[0][0], list) and
                    isinstance(references[0][0][0], str)
            ):
                multi_references = True

        # Get the encoder model
        model_name = self._get_model_name(model_type)
        encoder = _get_encoder(model_name, device=device)

        # Init output scores
        precisions = [0] * len(predictions)
        recalls = [0] * len(predictions)
        f1_scores = [0] * len(predictions)

        # Compute Score in case of single reference
        if not multi_references:
            for idx, (pred, ref) in enumerate(zip(predictions, references)):

                # Sentence Tokenize prediction and reference
                if tokenize_sentences:
                    ref = nltk.tokenize.sent_tokenize(ref)  # List[str]
                    pred = nltk.tokenize.sent_tokenize(pred)  # List[str]

                pred_sent_count = len(pred)
                embeddings = encoder.encode(pred + ref)
                pred_embeddings = embeddings[:pred_sent_count]
                ref_embeddings = embeddings[pred_sent_count:]

                p, r = _compute_cosine_similarity(pred_embeddings, ref_embeddings)
                f1 = _compute_f1(p, r)
                precisions[idx] = p
                recalls[idx] = r
                f1_scores[idx] = f1

        else:
            # Compute Score in case of multiple reference
            for idx, (pred, refs) in enumerate(zip(predictions, references)):
                # Sentence Tokenize prediction and reference
                if tokenize_sentences:
                    refs = [nltk.tokenize.sent_tokenize(ref) for ref in refs]  # List[List[str]]
                    pred = nltk.tokenize.sent_tokenize(pred)  # List[str]

                ref_count = len(refs)
                pred_sent_count = len(pred)
                ref_sent_counts = [0] + [len(ref) for ref in refs]
                cumsum_ref_sent_counts = np.cumsum(ref_sent_counts)

                all_sentences = pred + sum(refs, [])
                embeddings = encoder.encode(all_sentences)
                pred_embeddings = embeddings[:pred_sent_count]
                ref_embeddings = [
                    embeddings[pred_sent_count + cumsum_ref_sent_counts[c_idx]:
                               pred_sent_count + cumsum_ref_sent_counts[c_idx + 1]]
                    for c_idx in range(ref_count)
                ]
                # pred_embeddings = encoder.encode(pred)
                # ref_embeddings = [encoder.encode(refs) for ref in refs]

                # Precision: Concatenate all the sentences in all the references
                concat_ref_embeddings = np.concatenate(ref_embeddings, axis=0)
                p, _ = _compute_cosine_similarity(pred_embeddings, concat_ref_embeddings)

                # Recall: Compute individually for each reference
                scores = [_compute_cosine_similarity(r_embeds, pred_embeddings) for r_embeds in ref_embeddings]
                r = np.mean([r_scores for (r_scores, _) in scores]).item()

                f1 = _compute_f1(p, r)
                precisions[idx] = p  # TODO: check why idx says invalid type
                recalls[idx] = r
                f1_scores[idx] = f1

        return {"precision": precisions, "recall": recalls, "f1": f1_scores}