| import json |
| import os |
|
|
| import numpy as np |
|
|
| from evalplus.data import get_human_eval_plus, get_human_eval_plus_inputs |
|
|
| if __name__ == "__main__": |
| import argparse |
|
|
| parser = argparse.ArgumentParser() |
| parser.add_argument("--root", type=str, default="/JawTitan/EvalPlus/humaneval") |
| args = parser.parse_args() |
|
|
| plus_inputs = get_human_eval_plus_inputs() |
| problems = get_human_eval_plus().values() |
|
|
| base_bvs = {} |
| plus_bvs = {} |
| id2idx = {} |
|
|
| for i, problem in enumerate(problems): |
| task_id = problem["task_id"] |
| id2idx[task_id] = i |
| base_bvs[task_id] = np.zeros(len(problem["base_input"]), dtype=bool) |
| plus_bvs[task_id] = np.zeros(len(plus_inputs[task_id]), dtype=bool) |
|
|
| for path in os.listdir(args.root): |
| eval_json_path = os.path.join(args.root, path, "eval_results.json") |
| if not os.path.isfile(eval_json_path) or not path[-1].isdigit(): |
| print(f"skip {path}") |
| continue |
| res = json.load(open(eval_json_path, "r"))["eval"] |
|
|
| for task_id, v in res.items(): |
| for status, details in v["base"]: |
| if details is None: |
| continue |
| fails = np.logical_not(details) |
| base_bvs[task_id][: len(details)] = np.logical_xor( |
| base_bvs[task_id][: len(details)], fails |
| ) |
| for status, details in v["plus"]: |
| if details is None: |
| continue |
| fails = np.logical_not(details) |
| plus_bvs[task_id][: len(details)] = np.logical_xor( |
| plus_bvs[task_id][: len(details)], fails |
| ) |
|
|
| testsuite = [] |
|
|
| new_sizes = [] |
| for task_id, bbv in base_bvs.items(): |
| new_inputs = [] |
| idx = id2idx[task_id] |
| for i in np.nonzero(bbv)[0]: |
| new_inputs.append(problems[idx]["base_input"][i]) |
| pbv = plus_bvs[task_id] |
| for i in np.nonzero(pbv)[0]: |
| new_inputs.append(plus_inputs[task_id][i]) |
| testsuite.append({"task_id": task_id, "inputs": new_inputs}) |
| print( |
| task_id, f" org base {len(bbv)}; org plus {len(pbv)}; new {len(new_inputs)}" |
| ) |
| new_sizes.append(len(new_inputs)) |
|
|
| new_sizes = np.array(new_sizes) |
| print(f"{new_sizes.mean() = }, {new_sizes.min() = }, {new_sizes.max() = }") |
|
|
