evalplus/tools/humaneval/fix_v017.py

import math


def check_id(data, task_id):
    assert data[task_id]["task_id"] == f"HumanEval/{task_id}"


def fix(data):
    # https://github.com/evalplus/evalplus/issues/29
    check_id(data, 35)
    data[35]["contract"] += '    assert len(l) != 0, "invalid inputs" # $_CONTRACT_$\n'

    # https://github.com/evalplus/evalplus/issues/28
    check_id(data, 2)
    data[2][
        "contract"
    ] += '    assert number != float("+inf"), "invalid inputs" # $_CONTRACT_$\n'

    check_id(data, 99)
    data[99][
        "contract"
    ] += r"""    import math # $_CONTRACT_$
    assert not (math.isinf(value) or math.isnan(value)), "invalid inputs" # $_CONTRACT_$
"""
    # https://github.com/evalplus/evalplus/issues/27
    check_id(data, 1)
    data[1]["contract"] += '    assert cnt == 0, "invalid inputs" # $_CONTRACT_$\n'

    return data


if __name__ == "__main__":
    import json

    CONTRACT_INSPECT = [1, 2, 35, 99]
    SOURCE_VERSION = "v0.1.7"
    TARGET_VERSION = "v0.1.8"

    def evolve(src_file, tgt_file):
        with open(src_file) as f:
            data = [json.loads(line) for line in f.readlines() if line]

        data = fix(data)
        with open(tgt_file, "wb") as f:
            for x in data:
                f.write((json.dumps(x) + "\n").encode("utf-8"))

    evolve(
        f"HumanEvalPlus-{SOURCE_VERSION}.jsonl", f"HumanEvalPlus-{TARGET_VERSION}.jsonl"
    )
    evolve(
        f"HumanEvalPlus-Mini-{SOURCE_VERSION}.jsonl",
        f"HumanEvalPlus-Mini-{TARGET_VERSION}.jsonl",
    )

    # Debug the output of jsonl
    with open(f"HumanEvalPlus-{TARGET_VERSION}.jsonl") as f:
        data = [json.loads(line) for line in f.readlines() if line]

    data = {x["task_id"]: x for x in data}
    for tid in CONTRACT_INSPECT:
        print(data[f"HumanEval/{tid}"]["prompt"] + data[f"HumanEval/{tid}"]["contract"])
        print("====================================")