Spaces:
Runtime error
Runtime error
Commit
•
620ce47
1
Parent(s):
2697603
update
Browse files
src/backend/run_eval_suite.py
CHANGED
@@ -5,10 +5,7 @@ from src.backend.manage_requests import EvalRequest
|
|
5 |
|
6 |
from src.backend.tasks.xsum.task import XSum
|
7 |
from src.backend.tasks.cnndm.task import CNNDM
|
8 |
-
|
9 |
-
import logging
|
10 |
-
|
11 |
-
logging.getLogger("openai").setLevel(logging.WARNING)
|
12 |
|
13 |
|
14 |
def run_evaluation(eval_request: EvalRequest, task_names, num_fewshot, batch_size, device, use_cache=None, limit=None, max_nb_samples=100) -> dict:
|
|
|
5 |
|
6 |
from src.backend.tasks.xsum.task import XSum
|
7 |
from src.backend.tasks.cnndm.task import CNNDM
|
8 |
+
from src.backend.tasks.selfcheckgpt.task import SelfCheckGpt
|
|
|
|
|
|
|
9 |
|
10 |
|
11 |
def run_evaluation(eval_request: EvalRequest, task_names, num_fewshot, batch_size, device, use_cache=None, limit=None, max_nb_samples=100) -> dict:
|
src/backend/tasks/cnndm/task.py
CHANGED
@@ -141,6 +141,8 @@ class CNNDM(Task):
|
|
141 |
# all_refs = true_refs + false_refs
|
142 |
|
143 |
document = doc["article"]
|
|
|
|
|
144 |
true_refs = [doc["highlights"]]
|
145 |
all_refs = true_refs
|
146 |
|
|
|
141 |
# all_refs = true_refs + false_refs
|
142 |
|
143 |
document = doc["article"]
|
144 |
+
gold_summary = doc["highlights"]
|
145 |
+
|
146 |
true_refs = [doc["highlights"]]
|
147 |
all_refs = true_refs
|
148 |
|
src/backend/tasks/selfcheckgpt/task.py
CHANGED
@@ -1,12 +1,13 @@
|
|
1 |
import os
|
2 |
from typing import Union, List
|
3 |
|
4 |
-
|
5 |
from lm_eval.api.task import Task
|
6 |
from lm_eval.api.instance import Instance
|
7 |
from lm_eval.api.registry import register_task
|
8 |
from lm_eval.api.metrics import mean
|
9 |
|
|
|
|
|
10 |
import spacy
|
11 |
from selfcheckgpt.modeling_selfcheck import SelfCheckMQAG, SelfCheckNLI, SelfCheckBERTScore, SelfCheckNgram
|
12 |
|
@@ -16,7 +17,7 @@ class SelfCheckGpt(Task):
|
|
16 |
VERSION = 0.0
|
17 |
DATASET_PATH = "potsawee/wiki_bio_gpt3_hallucination"
|
18 |
DATASET_NAME = None
|
19 |
-
|
20 |
def __init__(self, data_dir=None, cache_dir=None, download_mode=None, config=None):
|
21 |
super().__init__(data_dir=data_dir, cache_dir=cache_dir, download_mode=download_mode, config=config)
|
22 |
self.generation_kwargs = {"temperature": 0.0, "do_sample": False}
|
@@ -24,7 +25,7 @@ class SelfCheckGpt(Task):
|
|
24 |
self.generation_kwargs_sampling = {"temperature": 1.0, "do_sample": False}
|
25 |
|
26 |
self.selfcheckgpt_type = os.environ.get('SELFCHECKGPTTYPE', 'SelfCheckNgram')
|
27 |
-
self.selfcheckgpt_device = os.environ.get('SELFCHECKGPTDEVICE',
|
28 |
self.selfcheckgpt_nlp = spacy.load("en_core_web_sm")
|
29 |
|
30 |
if self.selfcheckgpt_type == 'SelfCheckNgram':
|
@@ -59,34 +60,19 @@ class SelfCheckGpt(Task):
|
|
59 |
answer = doc['wiki_bio_text']
|
60 |
return answer
|
61 |
|
62 |
-
def construct_requests(
|
63 |
-
self, doc: dict, ctx: str, **kwargs
|
64 |
-
) -> Union[List[Instance], Instance]:
|
65 |
arguments = (ctx, self.generation_kwargs)
|
66 |
request_list = [
|
67 |
-
Instance(
|
68 |
-
request_type=self.OUTPUT_TYPE,
|
69 |
-
doc=doc,
|
70 |
-
arguments=arguments,
|
71 |
-
idx=0,
|
72 |
-
**kwargs
|
73 |
-
),
|
74 |
]
|
75 |
sampling_arguments = (ctx, self.generation_kwargs_sampling)
|
76 |
request_list.extend([
|
77 |
-
Instance(
|
78 |
-
request_type=self.OUTPUT_TYPE,
|
79 |
-
doc=doc,
|
80 |
-
arguments=sampling_arguments,
|
81 |
-
idx=idx,
|
82 |
-
**kwargs
|
83 |
-
)
|
84 |
for idx in range(1, self.generation_kwargs_sampling_number+1)
|
85 |
]
|
86 |
)
|
87 |
return request_list
|
88 |
|
89 |
-
|
90 |
def process_results(self, doc, results):
|
91 |
response_temperature_0 = results[0]
|
92 |
other_responses = results[1:]
|
@@ -104,24 +90,14 @@ class SelfCheckGpt(Task):
|
|
104 |
'max-selfcheckgpt': selfcheckgpt_scores['doc_level']['avg_max_neg_logprob']}
|
105 |
|
106 |
elif self.selfcheckgpt_type == 'SelfCheckBERTScore':
|
107 |
-
selfcheckgpt_scores = self.selfcheckgpt.predict(
|
108 |
-
sentences = sentences,
|
109 |
-
sampled_passages = other_responses,
|
110 |
-
)
|
111 |
elif self.selfcheckgpt_type == 'SelfCheckMQAG':
|
112 |
-
selfcheckgpt_scores = self.selfcheckgpt.predict(
|
113 |
-
sentences = sentences,
|
114 |
-
sampled_passages = other_responses,
|
115 |
-
)
|
116 |
elif self.selfcheckgpt_type == 'SelfCheckNLI':
|
117 |
-
selfcheckgpt_scores = self.selfcheckgpt.predict(
|
118 |
-
|
119 |
-
|
120 |
-
|
121 |
-
num_questions_per_sent = 5, # number of questions to be drawn
|
122 |
-
scoring_method = 'bayes_with_alpha', # options = 'counting', 'bayes', 'bayes_with_alpha'
|
123 |
-
beta1 = 0.8, beta2 = 0.8, # additional params depending on scoring_method
|
124 |
-
)
|
125 |
|
126 |
selfcheckgpt_scores_avg = sum(selfcheckgpt_scores) / len(selfcheckgpt_scores) if len(selfcheckgpt_scores) > 0 else 0
|
127 |
selfcheckgpt_scores_max = max(selfcheckgpt_scores)
|
|
|
1 |
import os
|
2 |
from typing import Union, List
|
3 |
|
|
|
4 |
from lm_eval.api.task import Task
|
5 |
from lm_eval.api.instance import Instance
|
6 |
from lm_eval.api.registry import register_task
|
7 |
from lm_eval.api.metrics import mean
|
8 |
|
9 |
+
from src.backend.envs import DEVICE
|
10 |
+
|
11 |
import spacy
|
12 |
from selfcheckgpt.modeling_selfcheck import SelfCheckMQAG, SelfCheckNLI, SelfCheckBERTScore, SelfCheckNgram
|
13 |
|
|
|
17 |
VERSION = 0.0
|
18 |
DATASET_PATH = "potsawee/wiki_bio_gpt3_hallucination"
|
19 |
DATASET_NAME = None
|
20 |
+
|
21 |
def __init__(self, data_dir=None, cache_dir=None, download_mode=None, config=None):
|
22 |
super().__init__(data_dir=data_dir, cache_dir=cache_dir, download_mode=download_mode, config=config)
|
23 |
self.generation_kwargs = {"temperature": 0.0, "do_sample": False}
|
|
|
25 |
self.generation_kwargs_sampling = {"temperature": 1.0, "do_sample": False}
|
26 |
|
27 |
self.selfcheckgpt_type = os.environ.get('SELFCHECKGPTTYPE', 'SelfCheckNgram')
|
28 |
+
self.selfcheckgpt_device = os.environ.get('SELFCHECKGPTDEVICE', DEVICE)
|
29 |
self.selfcheckgpt_nlp = spacy.load("en_core_web_sm")
|
30 |
|
31 |
if self.selfcheckgpt_type == 'SelfCheckNgram':
|
|
|
60 |
answer = doc['wiki_bio_text']
|
61 |
return answer
|
62 |
|
63 |
+
def construct_requests(self, doc: dict, ctx: str, **kwargs) -> Union[List[Instance], Instance]:
|
|
|
|
|
64 |
arguments = (ctx, self.generation_kwargs)
|
65 |
request_list = [
|
66 |
+
Instance(request_type='generate_until', doc=doc, arguments=arguments, idx=0, **kwargs),
|
|
|
|
|
|
|
|
|
|
|
|
|
67 |
]
|
68 |
sampling_arguments = (ctx, self.generation_kwargs_sampling)
|
69 |
request_list.extend([
|
70 |
+
Instance(request_type='generate_until', doc=doc, arguments=sampling_arguments, idx=idx, **kwargs)
|
|
|
|
|
|
|
|
|
|
|
|
|
71 |
for idx in range(1, self.generation_kwargs_sampling_number+1)
|
72 |
]
|
73 |
)
|
74 |
return request_list
|
75 |
|
|
|
76 |
def process_results(self, doc, results):
|
77 |
response_temperature_0 = results[0]
|
78 |
other_responses = results[1:]
|
|
|
90 |
'max-selfcheckgpt': selfcheckgpt_scores['doc_level']['avg_max_neg_logprob']}
|
91 |
|
92 |
elif self.selfcheckgpt_type == 'SelfCheckBERTScore':
|
93 |
+
selfcheckgpt_scores = self.selfcheckgpt.predict(sentences=sentences, sampled_passages=other_responses)
|
|
|
|
|
|
|
94 |
elif self.selfcheckgpt_type == 'SelfCheckMQAG':
|
95 |
+
selfcheckgpt_scores = self.selfcheckgpt.predict(sentences=sentences, sampled_passages=other_responses)
|
|
|
|
|
|
|
96 |
elif self.selfcheckgpt_type == 'SelfCheckNLI':
|
97 |
+
selfcheckgpt_scores = self.selfcheckgpt.predict(sentences=sentences, passage=response_temperature_0, sampled_passages=other_responses,
|
98 |
+
num_questions_per_sent=5, # number of questions to be drawn
|
99 |
+
scoring_method='bayes_with_alpha', # options = 'counting', 'bayes', 'bayes_with_alpha'
|
100 |
+
beta1=0.8, beta2=0.8) # additional params depending on scoring_method
|
|
|
|
|
|
|
|
|
101 |
|
102 |
selfcheckgpt_scores_avg = sum(selfcheckgpt_scores) / len(selfcheckgpt_scores) if len(selfcheckgpt_scores) > 0 else 0
|
103 |
selfcheckgpt_scores_max = max(selfcheckgpt_scores)
|