Repository: UKGovernmentBEIS/inspect_ai
Issue: [bug] task args are not preserved after KeyboardInterrupt of `eval_set`
Root cause: Resource Mishandling

The following script documents and reproduces the bug (running on OS X)

```python
import asyncio
import contextlib
import re
import signal
import uuid
from collections.abc import Generator
from types import FrameType

from inspect_ai import Task, eval_set, task
from inspect_ai._util.error import PrerequisiteError
from inspect_ai.dataset import MemoryDataset, Sample
from inspect_ai.log import read_eval_log
from inspect_ai.solver import Generate, TaskState, solver


def main():
    run_failing_scenario()
    print("\n\n========================================\n\n")
    run_successful_scenario()


def run_failing_scenario():
    log_dir = f"tmp/scratch_{str(uuid.uuid4())[:4]}"
    print(f"Using log directory: {log_dir}")

    def run_eval_set():
        eval_set(
            tasks=[mytask("foo"), mytask2("bar")],
            log_dir=log_dir,
            max_tasks=2,
        )

    print("===First we run the evalset and simulate a KeyboardInterrupt after 3 seconds===")
    try:
        with keyboard_interrupt(3):
            run_eval_set()
    except KeyboardInterrupt:
        pass

    print("===Second we run the evalset to completion===")
    run_eval_set()

    print(
        "===Now we try to rerun the evalset. We expect it to succeed instantly because both tasks are",
        "already completed. Instead, it raises PrerequisiteError.===",
    )
    try:
        run_eval_set()
    except PrerequisiteError as error:
        match = re.search(r"Existing log file '([^']+)'", str(error))
        assert match
        filename: str = match.group(1)

        print(
            "===If we check the problematic log file we see that the task args are empty, even though we",
            "passed task args 'foo' and 'bar' to each task respectively. That's why it thinks that the",
            "file is not associated with a task we are trying to run===",
        )
        log = read_eval_log(f"{log_dir}/{filename}")
        assert log.eval.task_args == {}

        print(str(error))


def run_successful_scenario():
    print(
        "===In the successful scenario we run the evalset twice in a row without cancelling. This time"
        "everything works as expected.==="
    )

    log_dir = f"tmp/scratch_{str(uuid.uuid4())[:4]}"
    print(f"Using log directory: {log_dir}")

    def run_eval_set():
        eval_set(
            tasks=[mytask("foo"), mytask2("bar")],
            log_dir=log_dir,
            max_tasks=2,
        )

    run_eval_set()
    run_eval_set()


@solver
def mysolver():
    async def solve(state: TaskState, generate: Generate):
        await asyncio.sleep(5)
        return state

    return solve


@task
def mytask(task_arg: str):
    return Task(
        dataset=MemoryDataset([Sample(input="")]),
        solver=[mysolver()],
    )


@solver
def mysolver2():
    async def solve(state: TaskState, generate: Generate):
        await asyncio.sleep(1)
        return state

    return solve


@task
def mytask2(task_arg: str):
    return Task(
        dataset=MemoryDataset([Sample(input="")]),
        solver=[mysolver2()],
    )


@contextlib.contextmanager
def keyboard_interrupt(seconds: int) -> Generator[None, None, None]:
    # note that this context manager is used to simulate a keyboard interrupt, but the same bug occurs if we trigger a KeyboardInterrupt manually via CTRL+C
    def handler(signum: int, frame: FrameType | None) -> None:
        raise KeyboardInterrupt

    original_handler = signal.signal(signal.SIGALRM, handler)
    signal.alarm(seconds)

    try:
        yield
    finally:
        signal.alarm(0)
        signal.signal(signal.SIGALRM, original_handler)


if __name__ == "__main__":
    main()

```

superseded by #805