File size: 10,804 Bytes
61f3f56
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
import argparse

import openai
import json
import os
from tqdm import tqdm
import pandas as pd
import numpy as np
from collections import Counter
import time



parser = argparse.ArgumentParser(description='ChatGPT-based QA evaluation.')
parser.add_argument('--mmvet_path')
parser.add_argument('--ckpt_name')
parser.add_argument('--result_path')
args = parser.parse_args()


openai.api_base = ""
openai.api_key = ''

gpt_model = "gpt-4-0613"


prompt = """Compare the ground truth and prediction from AI models, to give a correctness score for the prediction. <AND> in the ground truth means it is totally right only when all elements in the ground truth are present in the prediction, and <OR> means it is totally right when any one element in the ground truth is present in the prediction. The correctness score is 0.0 (totally wrong), 0.1, 0.2, 0.3, 0.4, 0.5, 0.6, 0.7, 0.8, 0.9, or 1.0 (totally right). Just complete the last space of the correctness score.

Question | Ground truth | Prediction | Correctness
--- | --- | --- | ---
What is x in the equation? | -1 <AND> -5 | x = 3 | 0.0
What is x in the equation? | -1 <AND> -5 | x = -1 | 0.5
What is x in the equation? | -1 <AND> -5 | x = -5 | 0.5
What is x in the equation? | -1 <AND> -5 | x = -5 or 5 | 0.5
What is x in the equation? | -1 <AND> -5 | x = -1 or x = -5 | 1.0
Can you explain this meme? | This meme is poking fun at the fact that the names of the countries Iceland and Greenland are misleading. Despite its name, Iceland is known for its beautiful green landscapes, while Greenland is mostly covered in ice and snow. The meme is saying that the person has trust issues because the names of these countries do not accurately represent their landscapes. | The meme talks about Iceland and Greenland. It's pointing out that despite their names, Iceland is not very icy and Greenland isn't very green. | 0.4
Can you explain this meme? | This meme is poking fun at the fact that the names of the countries Iceland and Greenland are misleading. Despite its name, Iceland is known for its beautiful green landscapes, while Greenland is mostly covered in ice and snow. The meme is saying that the person has trust issues because the names of these countries do not accurately represent their landscapes. | The meme is using humor to point out the misleading nature of Iceland's and Greenland's names. Iceland, despite its name, has lush green landscapes while Greenland is mostly covered in ice and snow. The text 'This is why I have trust issues' is a playful way to suggest that these contradictions can lead to distrust or confusion. The humor in this meme is derived from the unexpected contrast between the names of the countries and their actual physical characteristics. | 1.0
"""

# load metadata
# Download mm-vet.zip and `unzip mm-vet.zip` and change the path below
mmvet_path = args.mmvet_path
use_sub_set = False
decimal_places = 1  # number of decimal places to round to

if use_sub_set:
    bard_set_file = os.path.join(mmvet_path, "bard_set.json")
    with open(bard_set_file, 'r') as f:
        sub_set = json.load(f)
    sub_set_name = 'bardset'
    sub_set_name = sub_set_name + '_'
else:
    sub_set = None
    sub_set_name = ''

mmvet_metadata = os.path.join(mmvet_path, "mm-vet.json")
with open(mmvet_metadata, 'r') as f:
    data = json.load(f)

counter = Counter()
cap_set_list = []
cap_set_counter = []
len_data = 0
for id, value in data.items():
    if sub_set is not None and id not in sub_set:
        continue
    question = value["question"]
    answer = value["answer"]
    cap = value["capability"]
    cap = set(cap)
    counter.update(cap)
    if cap not in cap_set_list:
        cap_set_list.append(cap)
        cap_set_counter.append(1)
    else:
        cap_set_counter[cap_set_list.index(cap)] += 1

    len_data += 1

sorted_list = counter.most_common()
columns = [k for k, v in sorted_list]
columns.append("total")
columns.append("std")
columns.append('runs')
df = pd.DataFrame(columns=columns)

cap_set_sorted_indices = np.argsort(-np.array(cap_set_counter))
new_cap_set_list = []
new_cap_set_counter = []
for index in cap_set_sorted_indices:
    new_cap_set_list.append(cap_set_list[index])
    new_cap_set_counter.append(cap_set_counter[index])

cap_set_list = new_cap_set_list
cap_set_counter = new_cap_set_counter
cap_set_names = ["_".join(list(cap_set)) for cap_set in cap_set_list]

columns2 = cap_set_names
columns2.append("total")
columns2.append("std")
columns2.append('runs')
df2 = pd.DataFrame(columns=columns2)








###### change your model name ######
model = args.ckpt_name
result_path = args.result_path
num_run = 1 # we set it as 5 in the paper
model_results_file = os.path.join(result_path, f"{model}.json")

# grade results for each sample to svae
grade_file = f'{model}_{gpt_model}-grade-{num_run}runs.json'
grade_file = os.path.join(result_path, grade_file)

# score results regarding capabilities/capability integration to save
cap_score_file = f'{model}_{sub_set_name}{gpt_model}-cap-score-{num_run}runs.csv'
cap_score_file = os.path.join(result_path, cap_score_file)
cap_int_score_file = f'{model}_{sub_set_name}{gpt_model}-cap-int-score-{num_run}runs.csv'
cap_int_score_file = os.path.join(result_path, cap_int_score_file)

with open(model_results_file) as f:
    results = json.load(f)
if os.path.exists(grade_file):
    with open(grade_file, 'r') as f:
        grade_results = json.load(f)
else:
    grade_results = {}


def need_more_runs():
    need_more_runs = False
    if len(grade_results) > 0:
        for k, v in grade_results.items():
            if len(v['score']) < num_run:
                need_more_runs = True
                break
    return need_more_runs or len(grade_results) < len_data


while need_more_runs():
    for j in range(num_run):
        print(f'eval run {j}')
        for id, line in tqdm(data.items()):
            if sub_set is not None and id not in sub_set:
                continue
            if id in grade_results and len(grade_results[id]['score']) >= (j + 1):
                continue

            model_pred = results[id]

            question = prompt + '\n' + ' | '.join(
                [line['question'], line['answer'].replace("<AND>", " <AND> ").replace("<OR>", " <OR> "), model_pred,
                 ""])
            messages = [
                {"role": "user", "content": question},
            ]

            if id not in grade_results:
                sample_grade = {'model': [], 'content': [], 'score': []}
            else:
                sample_grade = grade_results[id]

            grade_sample_run_complete = False
            temperature = 0.0

            while not grade_sample_run_complete:
                try:
                    response = openai.ChatCompletion.create(
                        model=gpt_model,
                        max_tokens=3,
                        temperature=temperature,
                        messages=messages)
                    # print(response['model'])
                    content = response['choices'][0]['message']['content']
                    flag = True
                    try_time = 1
                    while flag:
                        try:
                            content = content.split(' ')[0].strip()
                            score = float(content)
                            if score > 1.0 or score < 0.0:
                                assert False
                            flag = False
                        except:
                            question = prompt + '\n' + ' | '.join(
                                [line['question'], line['answer'].replace("<AND>", " <AND> ").replace("<OR>", " <OR> "),
                                 model_pred, ""]) + "\nPredict the correctness of the answer (digit): "
                            messages = [
                                {"role": "user", "content": question},
                            ]
                            response = openai.ChatCompletion.create(
                                model=gpt_model,
                                max_tokens=3,
                                temperature=temperature,
                                messages=messages)
                            # print(response)
                            content = response['choices'][0]['message']['content']
                            try_time += 1
                            temperature += 0.5
                            print(f"{id} try {try_time} times")
                            print(content)
                            if try_time > 5:
                                score = 0.0
                                flag = False
                    grade_sample_run_complete = True
                except:
                    # gpt4 may have token rate limit
                    print("sleep 1s")
                    time.sleep(1)

            if len(sample_grade['model']) >= j + 1:
                sample_grade['model'][j] = response['model']
                sample_grade['content'][j] = content
                sample_grade['score'][j] = score
            else:
                sample_grade['model'].append(response['model'])
                sample_grade['content'].append(content)
                sample_grade['score'].append(score)
            grade_results[id] = sample_grade

            with open(grade_file, 'w') as f:
                json.dump(grade_results, f, indent=4)

assert not need_more_runs()
cap_socres = {k: [0.0] * num_run for k in columns[:-2]}
counter['total'] = len_data

cap_socres2 = {k: [0.0] * num_run for k in columns2[:-2]}
counter2 = {columns2[i]: cap_set_counter[i] for i in range(len(cap_set_counter))}
counter2['total'] = len_data

for k, v in grade_results.items():
    if sub_set is not None and k not in sub_set:
        continue
    for i in range(num_run):
        score = v['score'][i]
        caps = set(data[k]['capability'])
        for c in caps:
            cap_socres[c][i] += score

        cap_socres['total'][i] += score

        index = cap_set_list.index(caps)
        cap_socres2[cap_set_names[index]][i] += score
        cap_socres2['total'][i] += score

for k, v in cap_socres.items():
    cap_socres[k] = np.array(v) / counter[k] * 100

std = round(cap_socres['total'].std(), decimal_places)
total_copy = cap_socres['total'].copy()
runs = str(list(np.round(total_copy, decimal_places)))

for k, v in cap_socres.items():
    cap_socres[k] = round(v.mean(), decimal_places)

cap_socres['std'] = std
cap_socres['runs'] = runs
df.loc[model] = cap_socres

for k, v in cap_socres2.items():
    cap_socres2[k] = round(np.mean(np.array(v) / counter2[k] * 100), decimal_places)
cap_socres2['std'] = std
cap_socres2['runs'] = runs
df2.loc[model] = cap_socres2

df.to_csv(cap_score_file)
df2.to_csv(cap_int_score_file)
print(df)
print(df2)