|
|
import json |
|
|
import re |
|
|
import argparse |
|
|
|
|
|
def parse_args(): |
|
|
parser = argparse.ArgumentParser(description="Evaluate videorefer-bench-q.") |
|
|
parser.add_argument("--pred-path", default=r'', help="The path to file containing prediction.") |
|
|
args = parser.parse_args() |
|
|
return args |
|
|
|
|
|
|
|
|
def main(): |
|
|
all_sum = {} |
|
|
right_num = {} |
|
|
|
|
|
args = parse_args() |
|
|
data = [] |
|
|
for line in open(args.pred_path): |
|
|
d = json.loads(line) |
|
|
data.append(d) |
|
|
|
|
|
for i,d in enumerate(data): |
|
|
gt = d['Answer'] |
|
|
match = re.search(r'\(([A-Z])\)', gt) |
|
|
if match: |
|
|
gt = match.group(1) |
|
|
|
|
|
match = re.search(r'\(([A-Z])\)', d['pred']) |
|
|
if match: |
|
|
answer = match.group(1) |
|
|
else: |
|
|
match = re.search(r'([A-Z])\)', d['pred']) |
|
|
if match: |
|
|
answer = match.group(1) |
|
|
else: |
|
|
answer = d['pred'].replace('.','')[0] |
|
|
|
|
|
if d['type'] not in all_sum: |
|
|
all_sum[d['type']] = 0 |
|
|
right_num[d['type']] = 0 |
|
|
if answer.lower()==gt.lower(): |
|
|
right_num[d['type']]+=1 |
|
|
|
|
|
|
|
|
all_sum[d['type']]+=1 |
|
|
|
|
|
all_type_sum = 0 |
|
|
all_type_right = 0 |
|
|
for tp in all_sum.keys(): |
|
|
print('####### ',tp, ' #######') |
|
|
print('all num: ',all_sum[tp]) |
|
|
print('right num: ',right_num[tp]) |
|
|
print('accuracy: ', right_num[tp]/all_sum[tp]) |
|
|
all_type_sum+=all_sum[tp] |
|
|
all_type_right+=right_num[tp] |
|
|
|
|
|
print('####### average #######') |
|
|
print('all num: ',all_type_sum) |
|
|
print('right num: ',all_type_right) |
|
|
print('accuracy: ', all_type_right/all_type_sum) |
|
|
|
|
|
|
|
|
if __name__ == '__main__': |
|
|
main() |
