Spaces:
Running
on
A10G
Running
on
A10G
| import os | |
| import argparse | |
| import json | |
| from llava.eval.m4c_evaluator import EvalAIAnswerProcessor | |
| def parse_args(): | |
| parser = argparse.ArgumentParser() | |
| parser.add_argument('--dir', type=str, default="./playground/data/eval/vqav2") | |
| parser.add_argument('--ckpt', type=str, required=True) | |
| parser.add_argument('--split', type=str, required=True) | |
| return parser.parse_args() | |
| if __name__ == '__main__': | |
| args = parse_args() | |
| src = os.path.join(args.dir, 'answers', args.split, args.ckpt, 'merge.jsonl') | |
| test_split = os.path.join(args.dir, 'llava_vqav2_mscoco_test2015.jsonl') | |
| dst = os.path.join(args.dir, 'answers_upload', args.split, f'{args.ckpt}.json') | |
| os.makedirs(os.path.dirname(dst), exist_ok=True) | |
| results = [] | |
| error_line = 0 | |
| for line_idx, line in enumerate(open(src)): | |
| try: | |
| results.append(json.loads(line)) | |
| except: | |
| error_line += 1 | |
| results = {x['question_id']: x['text'] for x in results} | |
| test_split = [json.loads(line) for line in open(test_split)] | |
| split_ids = set([x['question_id'] for x in test_split]) | |
| print(f'total results: {len(results)}, total split: {len(test_split)}, error_line: {error_line}') | |
| all_answers = [] | |
| answer_processor = EvalAIAnswerProcessor() | |
| for x in test_split: | |
| if x['question_id'] not in results: | |
| all_answers.append({ | |
| 'question_id': x['question_id'], | |
| 'answer': '' | |
| }) | |
| else: | |
| all_answers.append({ | |
| 'question_id': x['question_id'], | |
| 'answer': answer_processor(results[x['question_id']]) | |
| }) | |
| with open(dst, 'w') as f: | |
| json.dump(all_answers, open(dst, 'w')) | |