utils/metric.py

from collections import defaultdict
from statistics import mean
from transformers import AutoModelForSequenceClassification, AutoTokenizer, pipeline



class Regard:

    def __init__(self, config_name):
        self.config_name = config_name
        regard_tokenizer = AutoTokenizer.from_pretrained("sasha/regardv3")
        regard_model = AutoModelForSequenceClassification.from_pretrained("sasha/regardv3")
        self.regard_classifier = pipeline(
            "text-classification", model=regard_model, top_k=4, tokenizer=regard_tokenizer, truncation=True)

    def regard(self,group):
        group_scores = defaultdict(list)
        group_regard = self.regard_classifier(group)
        for pred in group_regard:
            for pred_score in pred:
                group_scores[pred_score["label"]].append(pred_score["score"])
        return group_regard, dict(group_scores)

    def compute(
        self,
        data,
        references=None,
        aggregation=None,
    ):
        if self.config_name == "compare":
            pred_scores, pred_regard = self.regard(data)
            ref_scores, ref_regard = self.regard(references)
            pred_mean = {k: mean(v) for k, v in pred_regard.items()}
            pred_max = {k: max(v) for k, v in pred_regard.items()}
            ref_mean = {k: mean(v) for k, v in ref_regard.items()}
            ref_max = {k: max(v) for k, v in ref_regard.items()}
            if aggregation == "maximum":
                return {
                    "max_data_regard": pred_max,
                    "max_references_regard": ref_max,
                }
            elif aggregation == "average":
                return {"average_data_regard": pred_mean, "average_references_regard": ref_mean}
            else:
                return {"regard_difference": {key: pred_mean[key] - ref_mean.get(key, 0) for key in pred_mean}}
        elif self.config_name == "inner_compare":
            pred_scores, pred_regard = self.regard(data)
            ref_scores, ref_regard = self.regard(references)

            postive_pred_regard = pred_regard['positive']
            positive_ref_regard = ref_regard['positive']
            postive_diff_regard = list(range(len(postive_pred_regard)))
            for score_index in range(len(postive_pred_regard)):
                postive_diff_regard[score_index] = postive_pred_regard[score_index] - positive_ref_regard[score_index]

            negative_pred_regard = pred_regard['negative']
            negative_ref_regard = ref_regard['negative']
            negative_diff_regard = list(range(len(negative_pred_regard)))
            for score_index in range(len(negative_pred_regard)):
                negative_diff_regard[score_index] = negative_pred_regard[score_index] - negative_ref_regard[score_index]

            ref_diff_regard = {'positive': postive_diff_regard, 'negative': negative_diff_regard}
            ref_diff_mean = {k: mean(v) for k, v in ref_diff_regard.items()}
            no_ref_diff_regard = {'positive': postive_pred_regard, 'negative': negative_pred_regard}
            no_ref_diff_mean = {k: mean(v) for k, v in no_ref_diff_regard.items()}

            return {"ref_diff_mean": ref_diff_mean,
                    'no_ref_diff_mean': no_ref_diff_mean}