File size: 3,258 Bytes
44466c7 6be3feb 44466c7 6be3feb |
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 |
from collections import defaultdict
from statistics import mean
from transformers import AutoModelForSequenceClassification, AutoTokenizer, pipeline
class Regard:
def __init__(self, config_name):
self.config_name = config_name
regard_tokenizer = AutoTokenizer.from_pretrained("sasha/regardv3")
regard_model = AutoModelForSequenceClassification.from_pretrained("sasha/regardv3")
self.regard_classifier = pipeline(
"text-classification", model=regard_model, top_k=4, tokenizer=regard_tokenizer, truncation=True)
def regard(self,group):
group_scores = defaultdict(list)
group_regard = self.regard_classifier(group)
for pred in group_regard:
for pred_score in pred:
group_scores[pred_score["label"]].append(pred_score["score"])
return group_regard, dict(group_scores)
def compute(
self,
data,
references=None,
aggregation=None,
):
if self.config_name == "compare":
pred_scores, pred_regard = self.regard(data)
ref_scores, ref_regard = self.regard(references)
pred_mean = {k: mean(v) for k, v in pred_regard.items()}
pred_max = {k: max(v) for k, v in pred_regard.items()}
ref_mean = {k: mean(v) for k, v in ref_regard.items()}
ref_max = {k: max(v) for k, v in ref_regard.items()}
if aggregation == "maximum":
return {
"max_data_regard": pred_max,
"max_references_regard": ref_max,
}
elif aggregation == "average":
return {"average_data_regard": pred_mean, "average_references_regard": ref_mean}
else:
return {"regard_difference": {key: pred_mean[key] - ref_mean.get(key, 0) for key in pred_mean}}
elif self.config_name == "inner_compare":
pred_scores, pred_regard = self.regard(data)
ref_scores, ref_regard = self.regard(references)
postive_pred_regard = pred_regard['positive']
positive_ref_regard = ref_regard['positive']
postive_diff_regard = list(range(len(postive_pred_regard)))
for score_index in range(len(postive_pred_regard)):
postive_diff_regard[score_index] = postive_pred_regard[score_index] - positive_ref_regard[score_index]
negative_pred_regard = pred_regard['negative']
negative_ref_regard = ref_regard['negative']
negative_diff_regard = list(range(len(negative_pred_regard)))
for score_index in range(len(negative_pred_regard)):
negative_diff_regard[score_index] = negative_pred_regard[score_index] - negative_ref_regard[score_index]
ref_diff_regard = {'positive': postive_diff_regard, 'negative': negative_diff_regard}
ref_diff_mean = {k: mean(v) for k, v in ref_diff_regard.items()}
no_ref_diff_regard = {'positive': postive_pred_regard, 'negative': negative_pred_regard}
no_ref_diff_mean = {k: mean(v) for k, v in no_ref_diff_regard.items()}
return {"ref_diff_mean": ref_diff_mean,
'no_ref_diff_mean': no_ref_diff_mean}
|